catnat: Analyse neuronal morphology and connectivity (extends catmaid and nat)

Documented in catmaid_connector_nodes catmaid_controlled_upload catmaid_delete_connectors catmaid_delete_neuron catmaid_delete_neurons catmaid_duplicated catmaid_find_likely_merge catmaid_get_neuronid catmaid_get_server catmaid_get_tag catmaid_interactive_join catmaid_interactive_name_transfer catmaid_join_skeletons catmaid_link_connectors catmaid_lock_neurons catmaid_node_time catmaid_skeletons_in_bbx catmaid_unlock_neurons catmaid_update_radius catmaid_upload_neurons fafbseg_join_connectors_in_ngl_volumes fafbseg_transfer_connectors fafbseg_transfer_tags local_conn

#' Upload neuron(s) to CATMAID
#'
#' @description  Uploads neurons to CATMAID, names them and annotates them.
#' Please use with caution, as you could be heavily adding to a live tracing environment.
#' @param swc a neuron or neuronlist object, or (a) local file path(s) to your saved .swc file(s).
#' @param name whatever you want to name your uploaded neurons. If a single character, then it will be added to all uploaded neurons. Else, can be a character vector the same length as swc.
#' @param annotations a character vector of annotations, to be added to all of the uploaded neurons
#' @param return.new.skids if TRUE, the new skids created in the CATMAID instance specified by conn, are returned
#' @param include.tags whether of not, if swc is a CATMAID neuron/neuronlist, to transfer its tags to its newly uploaded cognate
#' @param include.connectors whether of not, if swc is a CATMAID neuron/neuronlist, to transfer its connectors to its newly uploaded cognate
#' @param pid project id. Defaults to 1
#' @param conn CATMAID connection object, see ?catmaid::catmaid_login for details
#' @param max.upload the maximum number of files that the function will allow you to upload at once
#' @param ... methods passed to catmaid::catmaid_fetch
#' @export
#' @rdname catmaid_upload_neurons
catmaid_upload_neurons <- function(swc = NULL, name ="neuron SWC upload", annotations = NULL,
                                   include.tags = TRUE,include.connectors = TRUE,
                                   return.new.skids = FALSE,
                                   pid = 1, conn = NULL, max.upload = 10, ...) {
  message("Upload location: ", catmaid_get_server(conn))
  annotations = c(annotations,"neuron upload")
  if(nat::is.neuronlist(swc)|nat::is.neuron(swc)){
    neurons = nat::as.neuronlist(swc)
    if(nat::is.neuron(swc)){
      temp.files = "1.swc"
      nat::write.neuron(swc, format= "swc",dir = tempdir(), file = temp.files, Force = TRUE)
    }else{
      temp.files = paste0(names(swc),".swc")
      nat::write.neurons(swc, format= "swc",dir = tempdir(), files = temp.files, Force = TRUE)
    }
    swc = paste0(tempdir(),"/",temp.files)
  }else{
    neurons = NULL
  }
  if (length(swc)>max.upload){
    stop(paste0('You are uploading a large number of SWC files.
                Are you sure you want to do this?
                If so change the value of the max.upload argument'))
  }
  if(!is.null(name)){
    if(length(name)==1&length(swc)>1){
      name = paste0(name,"_",seq_along(swc))
    } else if (length(name)>length(swc)|length(swc)>length(name)){
      stop(paste0('The names argument mut be a vector or length 1, to be applied to all neurons
                  or a vector the same length as swc'))
    }
    }
  skids = c()
  for(file in 1:length(swc)){
    post_data = list()
    post_data["name[1]"] = as.list(name[file])
    post_data["file[1]"] = list(upload_file(swc[file]))
    path = sprintf("/%d/skeletons/import", pid)
    res = catmaid::catmaid_fetch(path, body = post_data, include_headers = F,
                                 simplifyVector = T, conn = conn, ...)
    invisible(catmaid:::catmaid_error_check(res))
    catmaid::catmaid_set_annotations_for_skeletons(skids = res$skeleton_id,
                                                   annotations = annotations,
                                                   pid = pid, conn = conn, ...)
    tryCatch(catmaid::catmaid_rename_neuron(skids = res$skeleton_id, names = name[file], pid = pid, conn = conn, ...), error = function(e)
      warning("Could not name new neuron ",res$skeleton_id))
    message(paste0("Annotations and names set to new skeleton: ", res$skeleton_id))
    new.neuron = catmaid::read.neuron.catmaid(res$skeleton_id, pid = pid, conn = conn, ...)
    if(include.tags&nat::is.neuron(neurons[[file]])){
      fafbseg_transfer_tags(new.neuron = new.neuron, old.neuron = neurons[[file]],
                            pid = pid, conn = conn, search.range.nm = 0, offset = FALSE, ...)
    }
    if(include.connectors&nat::is.neuron(neurons[[file]])){
      fafbseg_transfer_connectors(new.neuron = new.neuron, old.neuron = neurons[[file]], links = TRUE,
                                  search.range.nm = 0, offset = FALSE, pid = pid, conn = conn, ...)
    }
    skids = c(skids,res$skeleton_id)
  }
  if(return.new.skids){
    skids
  }
}

#' Transfer information between v14 and v14-seg neurons
#'
#' @description  Functions to enable the mapping of connectors and tags between cognate FAFB v14 and v14-seg neurons, to help enable neuronal reconstruction using the partial auto-segmentation in v14-seg space
#' @param new.neuron a neuron object, or the skeleton ID of a neuron in v14 space
#' @param old.neuron a neuron object, or the skeleton ID of a neuron in v14-seg space
#' @param offset whether or not to expect a displacement in space between the new and old neurons. If so, search.range.nm is used to find the nearest nodes to which to assign tags/connectors, within the radius it specifies
#' @param search.range.nm the maximum distance between points in the v14 and v14-seg skeleton, that is acceptable for the transfer ot tag/connector information. Not set to 0 be default to accept small variations in node position that might occur between the two tracing environments, due to tracer edit.s
#' @param links whether or not to link transferred connectors  to post-pre synaptic sites on the neuron identified with the new.neuron argument
#' @param pid project id for conn. Defaults to 1
#' @param pid2 project id for conn2. Defaults to 1
#' @param conn CATMAID connection object, see ?catmaid::catmaid_login for details
#' @param conn2 CATMAID connection object, see ?catmaid::catmaid_login for details
#' @param ... methods passed to catmaid::catmaid_set_labels
#' @export
#' @rdname fafbseg_transfer
fafbseg_transfer_tags <- function(new.neuron, old.neuron, offset = FALSE, search.range.nm = 1000, pid = 1, conn = NULL, pid2 = 1, conn2 = fafb_seg_conn(), ...){
  if(!nat::is.neuron(new.neuron)){
    new.neuron = catmaid::read.neuron.catmaid(new.neuron, pid = pid, conn = conn, ...)
  }
  if(!nat::is.neuron(old.neuron)){
    old.neuron = catmaid::read.neuron.catmaid(new.neuron, pid = pid2, conn = conn2, ...)
  }
  if(!is.null(old.neuron$tags)&length(old.neuron$tags)){
    labels = unlist(sapply(1:length(old.neuron$tags),function(t) rep(names(old.neuron$tags[t]),length(old.neuron$tags[[t]]))))
    tagged.points = unlist(old.neuron$tags)
    tagged.points = tagged.points[tagged.points%in%old.neuron$d$PointNo]
    tagged.points = old.neuron$d[match(tagged.points,old.neuron$d$PointNo),]
    tagged.points$tag = labels
    near = tryCatch(nabor::knn(query = tagged.points[,c('X','Y', 'Z')], data =nat::xyzmatrix(new.neuron), k=1, radius= ifelse(offset,search.range.nm,0) )$nn.idx,
                    error = function(e) nabor::knn(query = tagged.points[,c('X','Y', 'Z')], data =nat::xyzmatrix(new.neuron), k=1)$nn.idx)
    near = near[near!=0]
    labels = tagged.points$tag[near!=0]
    nodes = new.neuron$d$PointNo[near]
    if(length(nodes)!=length(labels)){
      warning("error with assigning tag information")
    }
    if(length(nodes)>0){
      for(i in 1:length(nodes)){
        catmaid::catmaid_set_labels(node = nodes[i] , labels = labels[i], type= "treenode", pid = pid, conn = conn, ...)
      }
    }
  }
}
#' @export
#' @rdname fafbseg_transfer
fafbseg_transfer_connectors<- function(new.neuron, old.neuron,
                                       offset = FALSE, search.range.nm = 1000, links = TRUE,
                                       pid = 1, conn = NULL, pid2 = 1, conn2 = fafb_seg_conn(), ...){
  if(!nat::is.neuron(new.neuron)){
    new.neuron = catmaid::read.neuron.catmaid(new.neuron, pid = pid, conn = conn, ...)
  }
  if(!nat::is.neuron(old.neuron)){
    old.neuron = catmaid::read.neuron.catmaid(new.neuron, pid = pid2, conn = conn2, ...)
  }
  if(!is.null(old.neuron$connector)&length(old.neuron$connector)){
    c.df = old.neuron$connector
    near = tryCatch(nabor::knn(query = nat::xyzmatrix(c.df), data = nat::xyzmatrix(new.neuron), k=1, radius=ifelse(offset,search.range.nm,0))$nn.idx,
                      error = function(e) nnabor::knn(query = nat::xyzmatrix(c.df), data = nat::xyzmatrix(new.neuron), k=1)$nn.idx)
    new.c.df = c.df[near!=0,]
    new.c.df$near_id = new.neuron$d[near,"PointNo"]
    if(nrow(new.c.df)>0){
      for(i in 1:nrow(new.c.df)){
        post_data = list()
        post_data["x"] = new.c.df[i,"x"]
        post_data["y"] = new.c.df[i,"y"]
        post_data["z"] = new.c.df[i,"z"]
        path = sprintf("/%d/connector/create", pid)
        res = catmaid::catmaid_fetch(path, body = post_data, include_headers = F,
                                     simplifyVector = T, conn = conn, ...)
        invisible(catmaid:::catmaid_error_check(res))
        if(links){
          link_type = ifelse(new.c.df[i,"prepost"]==0,"presynaptic","postsynaptic")
          catmaid_link_connectors(treenode_id = new.c.df[i,"near_id"],
                                  connector_id = res$connector_id,
                                  link_type = link_type,
                                  pid = pid, conn = conn,...)
        }
      }
    }
  }
}

#' Interactively transfer names and annotations between neurons in two different CATMAID instances
#'
#' @description  Interactively transfer names and annotations between neurons in two different CATMAID instances
#' @param x a neuron object, or the skeleton ID of a neuron in v14 space
#' @param transfer.annotations whether or not to also transfer annotations
#' @param additional.annotations an additional annotation to add to the neurons specified by x
#' @param duplication.range.nm determines the size of the bounding box around each node in neuron to search for a duplicated. Defaults to 1 nm
#' @param downsample if downsample > 1, a sample of number.of.points/downsample is taken from the neuron, and only these points are used to assess duplication. Speeds things up
#' @param fafbseg whether or not to use fafbseg::read_brainmaps_meshes on the TODO tag locations and restrict possible duplication to skeletons within the search range and within the cognate auto-segmented volume
#' @param pid project id for conn. Defaults to 1
#' @param pid2 project id for conn2. Defaults to 1
#' @param conn CATMAID connection object, see ?catmaid::catmaid_login for details
#' @param conn2 CATMAID connection object, see ?catmaid::catmaid_login for details
#' @param ... methods passed to catmaid::catmaid_fetch
#' @export
#' @rdname catmaid_interactive_name_transfer
catmaid_interactive_name_transfer <- function(x = "name:ASB CHECK", transfer.annotations = TRUE,
                                              additional.annotations = "ASB CHECK",
                                              downsample = 100, fafbseg = FALSE, duplication.range.nm = 1,
                                              pid = 1, conn = NULL, pid2 = 1, conn2 = fafb_seg_conn(), ...){
  message("Reading neurons in CATMAID instance ", catmaid_get_server(conn))
  message("Reading: ", x)
  neurons1 = catmaid::read.neurons.catmaid(x, OmitFailures = TRUE, pid = pid, conn = conn, ...)
  message("Finding duplicate neurons in CATMAID instance ", catmaid_get_server(conn2))
  matches = c()
  matched =  c()
  for(i in 1:length(neurons1)){
    neuron = neurons1[[i]]
    message("Searching with ", names(neurons1)[i])
    dupe = catmaid_duplicated(neuron, tolerance = NULL, duplication.range.nm = duplication.range.nm,
                              downsample = downsample, fafbseg = fafbseg, pid = pid2, conn = conn2, ...)
    if(is.null(dupe$overlapping.skids)){
      message("No match found for ", names(neurons1)[i])
    }else{
      t = table(unlist(dupe$overlapping.skids))
      t = names(t)[order(t,decreasing = TRUE)][1]; t = t[!is.na(t)]
      matches = c(matches,t)
      matched = c(matched,names(neurons1)[i])
    }
  }
  neurons2 = neurons1[matched]
  nat::nopen3d()
  for(j in 1:length(neurons2)){
    rgl::clear3d()
    t = matches[j]
    o = matched[j]
    message("Considering changing the name of  ", o, " from ")
    print(neurons2[j,"name"])
    similar = catmaid::read.neurons.catmaid(t, OmitFailures = TRUE, pid = pid2, conn = conn2, ...)
    rgl::plot3d(neurons2[j], col= "black", lwd=2, WithConnectors = TRUE)
    rgl::plot3d(similar, col="grey", lwd=5, WithConnectors = TRUE)
    message("To its most overlapping neuron (grey):")
    print(similar[,"name"])
    continue = readline("Do you want to transfer this name? y=yes,n=no: ")
    if(continue=="y"){
      tryCatch(catmaid::catmaid_rename_neuron(skids = o, names = similar[,"name"], pid = pid, conn = conn, ...),
               error = function(e) warning("Could not rename neuron ",o))
    }else{
      message("Not changing name for  ", o)
    }
    if(transfer.annotations){
      anns2 = catmaid::catmaid_get_annotations_for_skeletons(t,pid=pid2,conn=conn2,...)
      print(anns2$annotation)
      continue2 = readline("Do you want to transfer these annotations? y=yes,n=no: ")
      if(continue2=="y"&!is.null(anns2$annotation)){
        anns = catmaid::catmaid_get_annotations_for_skeletons(o,pid=pid,conn=conn,...)
        print(anns$annotation)
        continue3 = readline("Do you want to first try to remove these annotations annotations, if you have permission? y=yes,n=no: ")
        if(continue3=="y"&!is.null(anns$annotation)){
          message("removing annotations...")
          remove.annotations = sapply(anns$annotation, function(a) tryCatch(catmaid::catmaid_remove_annotations_for_skeletons(skids = o,
                                                                                                                              annotations = a,
                                                                                                                              pid = pid, conn = conn, ...),
                                                                            error = function(e) warning("Could not remove annotation ", a," for neuron ", o)))
        }
        message("setting annotations...")
        add.annotations = sapply(anns2$annotation, function(a) tryCatch(catmaid::catmaid_set_annotations_for_skeletons(skids = o,
                                                                                                                       annotations = a,
                                                                                                                       pid = pid, conn = conn, ...),
                                                                        error = function(e) warning("Could not set annotation ", a," for neuron ", o)))
      }else{
        message("Not annotations set or removed for  ", o)
      }
    }
    tryCatch(catmaid::catmaid_set_annotations_for_skeletons(skids = o,
                                                            annotations = additional.annotations,
                                                            pid = pid, conn = conn, ...),
             error = function(e) warning("Could not set annotation ", additional.annotations," for neuron ", o))
  }
}

### Added to rcatmaid ###
#' Get the UTC creation / edit time for a CATMAID node
#'
#' @description Get the UTC creation / edit time for a CATMAID treenode or connector.
#' Useful for making 'state' arguments to be passed to other functions that edit data on a CATMAID server.
#' @param id a treenode or connector ID
#' @param time whether to return the creation_time or edition_time
#' @param pid project id. Defaults to 1
#' @param conn CATMAID connection object, see ?catmaid::catmaid_login for details
#' @param ... methods passed to catmaid::catmaid_set_labels
#' @export
#' @rdname catmaid_node_time
catmaid_node_time <- function(id, time = c("creation_time", "edition_time"), pid = 1, conn = NULL, ...){
  time = match.arg(time)
  id = as.numeric(id)
  post_data = list()
  post_data["node_ids"] = id
  path = sprintf("/%d/node/user-info", pid)
  res = catmaid::catmaid_fetch(path, body = post_data, include_headers = F,
                               simplifyVector = T, conn = conn, ...)
  if(!is.null(res$error)){
    stop(res$error)
  }else{
    res[[1]][[time]]
  }
}

#' Add synaptic links between connectors and treenodes
#'
#' @description Add links between connectors and treenodes to designate pre- and postsynaptic connections in CATMAID
#' @param treenode_id the treenode ID to/from which a link is to be made
#' @param connector_id the connector ID to/from which a link is to be made
#' @param link_type whether the link is presynaptic or postsynaptic
#' @param verbose whether or not to report if a link is successfully made
#' @param pid project id. Defaults to 1
#' @param conn CATMAID connection object, see ?catmaid::catmaid_login for details
#' @param ... methods passed to catmaid::catmaid_fetch
#' @export
#' @rdname catmaid_link_connectors
catmaid_link_connectors <- function(treenode_id, connector_id,
                                    link_type = c("presynaptic","postsynaptic"),
                                    verbose = TRUE,
                                    pid = 1, conn = NULL, ...){
  link_type = match.arg(link_type)
  link_type = paste0(link_type,"_to")
  ### Get states ###
  from_time = catmaid_node_time(id=treenode_id, time = "edition_time", pid=pid, conn=conn, ...)
  to_time = catmaid_node_time(id=connector_id, time = "edition_time", pid=pid, conn=conn, ...)
  ### Make link ###
  post_data = list()
  post_data["from_id"] = treenode_id
  post_data["to_id"] = connector_id
  post_data["link_type"] = link_type
  post_data["state"] = sprintf('[[%s, "%s"], [%s, "%s"]]',treenode_id, from_time, connector_id, to_time)
  path = sprintf("/%d/link/create", pid)
  res = catmaid::catmaid_fetch(path, body = post_data, include_headers = F,
                               simplifyVector = T, conn = conn, ...)
  invisible(catmaid:::catmaid_error_check(res))
  if(verbose){
    if(!is.null(res$error)){
      stop(res$error)
    }else{
      message(res$message, " linking treenode ", treenode_id," and connector ", connector_id)
    }
  }
}


#' Find the location of specified tags for a CATMAID neuron
#'
#' @description  Find the location of tags in a CATMAID neuron, either as URLs to the location of a TODO tag in CATMAID or as a data.frame reporting the location and skeleton treenode locations of specified tags.
#' @param x a neuron or neuronlist object
#' @param tag a single character specifying which tag to look for. Defaults to TODO
#' @param only.leaves whether or not to only return leaf nodes with the specified tag
#' @param url if TRUE (default) a list of URLs pertaining to specified tag locations are returned. If FALSE, a data.frame subsetted from x$d is returned, reporting treenode ID and X,Y,Z positions for specified tags
#' @param pid project id. Defaults to 1. For making the URL.
#' @param conn CATMAID connection object, see ?catmaid::catmaid_login for details. For making the URL.
#' @export
#' @rdname catmaid_get_tag
catmaid_get_tag<-function(x, tag = "TODO", url = FALSE, only.leaves = TRUE, conn = NULL, pid = 1) UseMethod("catmaid_get_tag")
catmaid_get_tag.neuron <- function(x, tag = "TODO", url = FALSE, only.leaves = TRUE, conn = NULL, pid = 1){
  TODO = unique(unlist(x$tags[[tag]]))
  if(only.leaves){
    TODO = TODO[TODO%in%x$d$PointNo[nat::endpoints(x)]]
  }
  if(is.null(TODO)){
    NULL
  }else if(length(TODO)){
    df = subset(x$d,PointNo%in%TODO)
    if(url){
      catmaid_url = paste0(catmaid_get_server(conn), "?pid=",pid)
      catmaid_url = paste0(catmaid_url, "&zp=", df[["Z"]])
      catmaid_url = paste0(catmaid_url, "&yp=", df[["Y"]])
      catmaid_url = paste0(catmaid_url, "&xp=", df[["X"]])
      catmaid_url = paste0(catmaid_url, "&tool=tracingtool")
      catmaid_url = paste0(catmaid_url, "&sid0=5&s0=0")
      invisible(catmaid_url)
    }
    else{
      df
    }
  }
}
catmaid_get_tag.neuronlist <- function(x, tag = "TODO", url = FALSE, only.leaves = TRUE, conn = NULL, pid = 1){
  if(url){
    unlist(lapply(x,catmaid_get_tag.neuron, url=url, tag= tag))
  }else{
    do.call(rbind,lapply(x,catmaid_get_tag.neuron, url=url, tag = tag))
  }
}

#' Find the TODO tagged merge sites between a given neuron and the CATMAID database at large
#'
#' @description  Find the location of tags in a CATMAID neuron, either as URLs to the location of a TODO tag in CATMAID or as a data.frame reporting the location and skeleton treenode locations of specified tags.
#' @param TODO a data.frame, as returned by catmaid_get_tag with tag = "TODO" (or a different tag indicative of a merge point) and url = FALSE
#' @param skid the skeleton ID of a neuron for which merges are to be found. If NULL, the skeleton ID is taken as the rowname of the TODO data frame
#' @param fafbseg whether or not to use fafbseg::read_brainmaps_meshes on the TODO tag locations and restrict possible merges to neurons within the search range and within the cognate auto-segmented volume
#' @param search.range.nm the maximum distance from which to search from the TODO point tag to find potential mergers
#' @param min_nodes the minimum number of nodes a potential merger skeleton needs to have
#' @param pid project id. Defaults to 1
#' @param conn CATMAID connection object, see ?catmaid::catmaid_login for details
#' @param ... methods passed to catmaid::catmaid_fetch
#' @export
#' @rdname catmaid_find_likely_merge
catmaid_find_likely_merge <- function(TODO, skid = NULL, fafbseg = FALSE, min_nodes = 2,
                                      search.range.nm = 1000, pid=1, conn = conn, ...){
  possible.merges = data.frame()
  for(i in 1:nrow(TODO)){
    todo = TODO[i,]
    if(is.null(skid)){
      skid = floor(as.numeric(rownames(todo)))
    }
    bbx = rbind(todo[,c('X','Y', 'Z')]-search.range.nm,todo[,c('X','Y', 'Z')]+search.range.nm)
    skids.bbx = catmaid_skeletons_in_bbx(bbx=bbx, min_nodes=min_nodes, pid=pid, conn = conn, ...)
    neuron.bbx = catmaid::read.neurons.catmaid(skids.bbx, OmitFailures = TRUE, pid=pid, conn = conn, ...)
    neuron.bbx = neuron.bbx[setdiff(names(neuron.bbx),skid)]
    message(length(neuron.bbx), " fragments found within ", search.range.nm," nm of merge tag ", i)
    if(fafbseg){
      if(!requireNamespace('fafbseg', quietly = TRUE))
        stop("Please install suggested fafbseg package")
      message("Checking whether potential merges are within the same FAFB volumetric auto-traced segments")
      seg = tryCatch(fafbseg::brainmaps_xyz2id(todo[,c('X','Y', 'Z')]),error=function(e)NULL)
      seg = seg[seg!=0]
      s = tryCatch(fafbseg::find_merged_segments(seg),error=function(e)seg)
      s = s[s!=0]
      vol = tryCatch(suppressMessages(fafbseg::read_brainmaps_meshes(s)), error = function(e) message("warning: Google brainmaps read error, take extra care when deciding on join"))
      in.vol = tryCatch(sapply(neuron.bbx,function(n) sum(nat::pointsinside(nat::xyzmatrix(n),surf=vol))>0),error = function(e) rep(TRUE,length(neuron.bbx)))
      if(length(in.vol)){
        neuron.bbx = neuron.bbx[in.vol]
      }
      message(length(neuron.bbx), " fragments found in FAFB Google auto-traced segment corresponding with merge tag ", i)
    }
    if(length(neuron.bbx)){
      neuron.bbx.d = do.call(rbind,lapply(1:length(neuron.bbx),function(n) cbind(neuron.bbx[[n]]$d,skid=names(neuron.bbx[n]))))
      near = tryCatch(nabor::knn(query = todo[,c('X','Y', 'Z')], data = nat::xyzmatrix(neuron.bbx.d),k=10, radius=search.range.nm),
                      error = function(e) nabor::knn(query = todo[,c('X','Y', 'Z')], data = nat::xyzmatrix(neuron.bbx.d),k=10))
      near$nn.idx[near$nn.dists>=search.range.nm] = 0
      near = near$nn.idx
      near = near[near!=0]
      near = neuron.bbx.d[near,]
      near$TODO = todo$PointNo
      near$merger = skid
      #near = subset(possible.merges,upstream.skid!=skid)
      if(nrow(near)){
        possible.merges = plyr::rbind.fill(possible.merges,near)
      }
    }
  }
  possible.merges = possible.merges[,setdiff(colnames(possible.merges),"Parent")]
  colnames(possible.merges) = c("upstream.node","label","X","Y","Z","W","upstream.skid","downstream.node","downstream.skid")
  possible.merges
}

#' Interactively choose to join neurons in CATMAID, visualising with rgl in R
#'
#' @description Interactively choose to join neurons in CATMAID via rgl in R. Use with caution in live tracing environments.
#' @param possible.merges a data frame of possible mergers between tree nodes for neurons in a CATMAID database, as returned via catmaid_find_likely_merge
#' @param downstream.neurons neurons (typically smaller, new tracings) to be joined into other (typically larger) neurons, which will be pulled from CATMAID. If NULL, the downstream neurons are also pulled from CATMAID.
#' @param brain the brain to plot while visualising potential mergers using rgl. Defaults to NULL, no brain plotted.
#' @param pid project id. Defaults to 1
#' @param conn CATMAID connection object, see ?catmaid::catmaid_login for details
#' @param ... methods passed to catmaid::catmaid_fetch
#' @export
#' @rdname catmaid_interactive_join
catmaid_interactive_join <- function(possible.merges, downstream.neurons = NULL,
                                     brain = NULL, pid = 1, conn = conn, ...){
  possible.merges$downstream.nodes  = catmaid::catmaid_get_node_count(possible.merges$downstream.skid)
  possible.merges$upstream.nodes  = catmaid::catmaid_get_node_count(possible.merges$upstream.skid)
  if(!is.null(downstream.neurons)&!nat::is.neuronlist(downstream.neurons)){
    stop("downstream.neurons must either be a neuronlist,
         or left NULL if you want to fetch them from CATMAID within this function")
  }
  neurons = catmaid::read.neurons.catmaid(skids=unique(possible.merges$upstream.skid), OmitFailures = TRUE, pid=pid,conn=conn,...)
  last.merge.size = 1e6
  for(todo in 1:length(unique(possible.merges$downstream.node))){
    TODO.possible = subset(possible.merges,downstream.node==unique(possible.merges$downstream.node)[todo])
    skid = unique(TODO.possible[,"downstream.skid"])[todo]
    todo = unique(possible.merges$downstream.node)[todo]
    TODO.possible = subset(TODO.possible, upstream.skid!=skid)
    if(!is.null(downstream.neurons)){
      neuron = downstream.neurons[as.character(skid)]
    }else{
      neuron = catmaid::read.neurons.catmaid(skid, OmitFailures = TRUE, pid = pid, conn = conn, ...)
    }
    downstream.node = catmaid::catmaid_get_treenodes_detail(tnids = todo, pid = pid, conn = conn, ...)
    continue = FALSE
    multiple.joins = FALSE # First merge retains name and skid of the manual neuron, rather than the larger
    i = 1
    while(!continue){
      message("Option ",i," of ", nrow(TODO.possible), " for ", skid)
      rgl::clear3d()
      if(!is.null(brain)){
        rgl::plot3d(brain,alpha=0.1,col="grey")
      }
      merger = catmaid::catmaid_get_treenodes_detail(tnids = TODO.possible[i,"upstream.node"], pid = pid, conn = conn, ...)
      merger.neuron = neurons[as.character(merger$skid)]
      rgl::plot3d(neuron,col="red",lwd=2,soma=T)
      rgl::plot3d(merger.neuron,col="black",lwd=2,soma=T)
      rgl::spheres3d(nat::xyzmatrix(downstream.node), col="orange", radius = 10)
      rgl::spheres3d(nat::xyzmatrix(TODO.possible[i,]), col="grey", radius = 10)
      message(paste(merger.neuron[,],collapse = " "))
      catmaid_url = paste0(catmaid_get_server(conn), "?pid=",pid)
      catmaid_url = paste0(catmaid_url, "&zp=", TODO.possible[i,"Z"])
      catmaid_url = paste0(catmaid_url, "&yp=", TODO.possible[i,"Y"])
      catmaid_url = paste0(catmaid_url, "&xp=", TODO.possible[i,"X"])
      catmaid_url = paste0(catmaid_url, "&tool=tracingtool")
      catmaid_url = paste0(catmaid_url, "&active_skeleton_id=", skid)
      catmaid_url = paste0(catmaid_url, "&sid0=5&s0=0")
      message("See merge site in CATMAID: ", catmaid_url)
      progress = readline("Make merge? y = yes, n = no, c = cycle : ")
      if(progress=="y"){
        nat::npop3d()
        rgl::plot3d(merger.neuron,col="green",lwd=2,soma=T)
        sure = readline("Sure? y = yes, n = no : ")
        if(sure=="y"){
          # find out which skeleton is larger
          if(multiple.joins){
            message("multiple joins for the same uploaded fragment being made")
            if(last.merge.size>TODO.possible[i,"upstream.nodes"]){
              from_treenode_id = TODO.possible[i,"upstream.node"]
              to_treenode_id = TODO.possible[i,"downstream.node"]
            }else{
              from_treenode_id = TODO.possible[i,"downstream.node"]
              to_treenode_id = TODO.possible[i,"upstream.node"]
            }
            last.merge.size = catmaid::catmaid_get_node_count(as.character(merger$skid)) + unique(TODO.possible$downstream.nodes)
            catmaid_join_skeletons(from_treenode_id = from_treenode_id, to_treenode_id = to_treenode_id, pid = pid, conn = conn, ...)
          }else{
            last.merge.size = catmaid::catmaid_get_node_count(as.character(merger$skid)) + unique(TODO.possible$downstream.nodes)
            catmaid_join_skeletons(from_treenode_id = TODO.possible[i,"downstream.node"],
                                   to_treenode_id = TODO.possible[i,"upstream.node"],
                                   pid = pid, conn = conn, ...)
          }
          multiple.joins = TRUE
          continue=TRUE
        }else{
          i = ifelse(i==nrow(TODO.possible),i,i+1)
        }
      }else if(progress=="n"){
        TODO.possible = TODO.possible[-i,]
        if(nrow(TODO.possible)==0){
          continue = TRUE
        }else if (i > nrow(TODO.possible)){
          i = nrow(TODO.possible)
        }else{
          i = ifelse(i==nrow(TODO.possible),1,i+1)
        }
      }else{
        i = ifelse(i==nrow(TODO.possible),1,i+1)
      }
    }
  }
}

### Moved to rcatmaid ###
#' Get information on a CATMAID connector node
#'
#' @description Get information on a CATMAID connector node
#' @param connector_id a data frame of possible mergers between tree nodes for neurons in a CATMAID database, as returned via catmaid_find_likely_merge
#' @param node the brain to plot while visualising potential mergers using rgl. Defaults to NULL, no brain plotted.
#' @param pid project id. Defaults to 1
#' @param conn CATMAID connection object, see ?catmaid::catmaid_login for details
#' @param ... methods passed to catmaid::catmaid_fetch and catmaid::catmaid_get_treenode_detail
#' @export
#' @rdname catmaid_connector_nodes
catmaid_connector_nodes <- function(connector_id, node = c("presynaptic","postsynaptic"),
                                    pid=1, conn = conn, ...){
  node = match.arg(node)
  connector_id = as.numeric(connector_id)
  if(length(connector_id)!=1){
    stop("connector_id must be a single connector_id")
  }
  post_data = list()
  post_data["connector_ids[0]"] = connector_id
  path = sprintf("/%d/connector/skeletons", pid)
  res = catmaid::catmaid_fetch(path, body = post_data, include_headers = F,
                               simplifyVector = T, conn = conn,...)
  tnids = res[[1]][[2]][[paste0(node,"_to_node")]]
  detail = catmaid::catmaid_get_treenodes_detail (tnids=tnids, pid = pid, conn = conn, ...)
  detail$connector_id = connector_id
  detail
}

#' Join free connectors in a CATMAID instance to skeletons if they share a Google Brainmaps volume
#'
#' @description Join free connectors (the up or downstream nodes of a connector are joined, but only where this point belongs to a skeleton of 1 node) in a CATMAID instance to skeletons if they share a Google Brainmaps volume
#' @param x a neuronlist object or skeletons IDs / names / annotations that can be read by catmaid::catmaid_skids
#' @param direction whether to seek to assign connectors upstream of neurons specified by maybe.connected, or connections downstream, or both.
#' @param maybe.connected the skeleton IDs for neurons whose connectors (both incoming and outgoing) may be considered
#' @param connector.range.nm the range in nm within which to search for a treenode ID to attach the connector
#' @param node.match number of nodes a neuron from x must have within a brainmaps 3D volume, for the volume to be considered to belong to this neuron
#' @param pid project id. Defaults to 1
#' @param conn CATMAID connection object, see ?catmaid::catmaid_login for details
#' @param ... methods passed to catmaid::catmaid_fetch and catmaid::catmaid_get_treenode_detail
#' @return makes joins in the specified CATMAID instance, but also returns a list of URLs for the locations at which joins have been made
# #' @export
#' @rdname fafbseg_join_connectors_in_ngl_volumes
fafbseg_join_connectors_in_ngl_volumes <- function(x,
                                                   direction = c("presynapses","postsynapses", "both"),
                                                   maybe.connected,
                                                   connector.range.nm = 1000,
                                                   node.match=5,
                                                   pid=1,conn = NULL, ...){
  if(!requireNamespace('pbapply', quietly = TRUE))
    stop("Please install suggested pbapply package")
  if(!requireNamespace('fafbseg', quietly = TRUE))
    stop("Please install suggested fafbseg package")
  direction = match.arg(direction)
  maybe.connected = catmaid::catmaid_skids(x = maybe.connected,pid=pid,conn=conn,...)
  if(!nat::is.neuronlist(x)){
    message("Reading neurons from ", catmaid_get_server(conn))
    neurons = catmaid::read.neurons.catmaid(x, OmitFailures = TRUE, pid=pid,conn=conn, ...)
  }else{
    neurons = x
  }
  total.joins = c()
  ### Get ngl IDs corresponding to potential presynapses ###
  if(direction%in%c("presynapses","both")){
    message("Fetching putatively connected presynapses in ", catmaid_get_server(conn))
    connectors.pre = do.call(rbind,pbapply::pblapply(maybe.connected, function(pcs)
      tryCatch(catmaid::catmaid_get_connector_table(skids=pcs,
                                                    direction = "incoming",
                                                    get_partner_nodes = TRUE,
                                                    pid = pid, conn = conn, ...),
               error = function(e) NULL)))
    connectors.pre = connectors.pre[apply(connectors.pre,1,function(r) sum(is.na(r))==0),]
    connectors.pre = connectors.pre[!duplicated(connectors.pre$connector_id),]
    connectors.pre[is.na(connectors.pre$partner_skid)&is.na(connectors.pre$partner_nodes),"partner_nodes"] = 0
    connectors.pre = subset(connectors.pre,partner_nodes<10)
    message("Assigning FAFB segmented volumes to connector locations")
    connector.pre.segs = fafbseg::brainmaps_xyz2id(nat::xyzmatrix(connectors.pre))
    message("Identifying treenodes associated with presynapses...")
    partner_treenode_id = pbapply::pbsapply(1:nrow(connectors.pre), function(i)
      tryCatch(catmaid_connector_nodes(connector_id = connectors.pre[i,"connector_id"],node = "presynaptic",
                                       pid=pid,conn=conn, ...)$treenode_id,
               error = function(e) NA))
    if(nrow(connectors.pre)!=length(partner_treenode_id)){
      stop("Issues finding treenodes upstream of connector objects")
    }
    df.pre.connectors = data.frame(connector_id = connectors.pre$connector_id, connector.skid = connectors.pre$skid,
                                   x = connectors.pre$x, y = connectors.pre$y, z = connectors.pre$z,
                                   ngl_id = connector.pre.segs, link_type = "presynaptic",
                                   partner_nodes = connectors.pre$partner_nodes, partner_treenode_id = partner_treenode_id)
  }
  ### Get ngl IDs corresponding to potential postsynapses ###
  if(direction%in%c("postsynapses","both")){
    message("Fetching putatively connected postsynapses in ", catmaid_get_server(conn))
    connectors.post = do.call(rbind,pbapply::pblapply(maybe.connected, function(pcs)
      tryCatch(catmaid::catmaid_get_connector_table(skids=pcs,
                                                    direction = "outgoing",
                                                    get_partner_nodes = TRUE,
                                                    pid = pid, conn = conn, ...),
               error = function(e) NULL)))
    connectors.post = connectors.post[apply(connectors.post,1,function(r) sum(is.na(r))==0),]
    connectors.post = connectors.post[!duplicated(connectors.post$connector_id),]
    connectors.post[is.na(connectors.post$partner_skid)&is.na(connectors.post$partner_nodes),"partner_nodes"] = 0
    connectors.post = subset(connectors.post,partner_nodes<10&partner_nodes>0)
    connectors.post.nodes = data.frame()
    message("Identifying treenodes associated with postsynapses...")
    pb <- utils::txtProgressBar(min = 0, max = nrow(connectors.post), style = 3)
    for(i in 1:nrow(connectors.post)){
      node.df = catmaid_connector_nodes(connector_id = connectors.post[i,"connector_id"],node = "postsynaptic",pid=pid,conn=conn,...)
      node.df = merge(node.df[,c("connector_id","x","y","z","treenode_id")],
                      connectors.post[i,setdiff(colnames(connectors.post),c("x","y","z"))])
      connectors.post.nodes = rbind(connectors.post.nodes,node.df)
      utils::setTxtProgressBar(pb, i)
    }
    close(pb)
    partner_nodes = catmaid::catmaid_get_node_count(connectors.post.nodes$partner_skid, conn=conn, ...)
    connector.post.segs = fafbseg::brainmaps_xyz2id(nat::xyzmatrix(connectors.post.nodes))
    df.post.connectors = data.frame(connector_id = connectors.post.nodes$connector_id, connector.skid = connectors.post.nodes$skid,
                                    x = connectors.post.nodes$x, y = connectors.post.nodes$y, z = connectors.post.nodes$z,
                                    ngl_id = connector.post.segs, link_type = "postsynaptic",
                                    partner_nodes = partner_nodes, partner_treenode_id = connectors.post.nodes$treenode_id)
  }
  ### Assign to volumes
  if(direction=="both"){
    df.connectors = rbind(df.pre.connectors,df.post.connectors)
  }else if(direction%in%c("presynapses","both")){
    df.connectors = df.pre.connectors
  }else if(direction%in%c("postsynapses","both")){
    df.connectors = df.post.connectors
  }
  df.connectors = subset(df.connectors,ngl_id!=0)
  df.connectors=df.connectors[!is.na(df.connectors$partner_treenode_id)|!is.na(df.connectors$partner_nodes),]
  ### Get ngl IDs corresponding to the skeletons ###
  message("Finding the 3D segments that correspond to ", length(neurons), " neurons:")
  segs = map_fafbsegs_to_neuron(neurons, node.match = node.match)
  segs = subset(segs,ngl_id!=0)
  df = merge(segs,df.connectors,all.x = FALSE, all.y = FALSE)
  df = dplyr::distinct(df)
  ### For connectors and neurons in the same seg, find nearest treenode to connectors ###
  if(nrow(df)){
    for(i in 1:nrow(df)){
      neuron = neurons[df[i,"skid"]][[1]]
      # Get target node
      tnid2 = df[i,"partner_treenode_id"]
      tnid2.detail = catmaid::catmaid_get_treenodes_detail (tnids=tnid2, pid = pid, conn = conn, ...)
      if(is.null(tnid2.detail)){
        tnid2.detail = df[i,]
      }
      # Get nearest node on putative partner
      near = nabor::knn(data = nat::xyzmatrix(neuron), query = nat::xyzmatrix(tnid2.detail),k=1,radius = connector.range.nm)$nn.idx
      tnid = neuron$d[near,"PointNo"]
      ### Make links ###
      if(length(tnid)==1){
        edist = euc.dist(df[i,c("x","y","z")],tnid2.detail[,c("x","y","z")])
        if(edist<=connector.range.nm){
          catmaid_url = paste0(catmaid_get_server(conn), "?pid=",pid)
          catmaid_url = paste0(catmaid_url, "&zp=", df[i,"z"])
          catmaid_url = paste0(catmaid_url, "&yp=", df[i,"y"])
          catmaid_url = paste0(catmaid_url, "&xp=", df[i,"x"])
          catmaid_url = paste0(catmaid_url, "&tool=tracingtool")
          catmaid_url = paste0(catmaid_url, "&active_node_id=", df[i,"skid"])
          catmaid_url = paste0(catmaid_url, "&sid0=5&s0=0")
          message("Attempting join at: ", catmaid_url)
          total.joins = c(total.joins,catmaid_url)
          if(df[i,"partner_nodes"]==0){
            ## Make sure connector is still not connected to anything else!
            connector.info = catmaid_connector_nodes(connector_id = df[i,"connector_id"],node = df[i,"link_type"],
                                                     pid=pid,conn=conn, ...)
            if(connector.info$partner_nodes==0){
              tryCatch(catmaid_link_connectors(treenode_id = tnid, connector_id = df[i,"connector_id"],
                                               link_type = df[i,"link_type"],
                                               pid = pid, conn = conn, verbose = FALSE, ...),
                       error = function(e) message("treenode join ",tnid," to connector ", df[i,"connector_id"], "failed"))
            }
          }else if(df[i,"partner_nodes"]<10){
            ## Make sure tnid2 is still not connected to anything else!
            n = catmaid::catmaid_get_node_count(tnid2.detail$skid, conn=conn, OmitFailures = FALSE,...)
            if(n<10&length(tnid2)==1&length(tnid)==1){
              tryCatch(catmaid_join_skeletons(from_treenode_id = tnid2, to_treenode_id = tnid, pid = pid, conn = conn, ...),
                       error = function(e) message("treenode join ",tnid," to ", tnid2, " failed"))
            }else{
              message("treenode join ",tnid," to ", tnid2, " failed, possibly because ", tnid2, " is now connected to a different skeleton")
            }
          }
        }else{
          message("treenode join ",tnid," to ", tnid2, "failed, possibly because ", tnid2, " they  are too far apart")
        }
      }
    }
  }
  total.joins
}

#' Programmatically join CATMAID skeletons
#'
#' @description  Programmatically join CATMAID skeletons. Use with caution in live tracing environments.
#' @param from_treenode_id the treenode ID of the downstream neuron
#' @param to_treenode_id the treenode ID of the upstream neuron
#' @param pid project id. Defaults to 1
#' @param conn CATMAID connection object, see ?catmaid::catmaid_login for details
#' @param ... methods passed to catmaid::catmaid_fetch
#' @export
#' @rdname catmaid_join_skeletons
catmaid_join_skeletons <- function(from_treenode_id, to_treenode_id, pid = 1, conn = NULL, ...){
  post_data = list()
  post_data["from_id"] = to_treenode_id
  post_data["to_id"] = from_treenode_id
  path = sprintf("/%d/skeleton/join", pid)
  res = catmaid::catmaid_fetch(path, body = post_data, include_headers = F,
                               simplifyVector = T, conn = conn, ...)
  invisible(catmaid:::catmaid_error_check(res))
  message(res$message)
}

### moved to rcatmaid ###
#' Search for CATMAID skeletons within a volume
#'
#' @description  Programmatically search for skeleton IDs pertaining to neurons within a search volume defined by a bounding box.
#' @param bbx the bounding box (a matrix of 2 rows and 3 columns) describing a search volume
#' @param min_nodes the minimum number of nodes a neuron in the search area must have (includes nodes outside search area)
#' @param pid project id. Defaults to 1
#' @param conn CATMAID connection object, see ?catmaid::catmaid_login for details
#' @param ... methods passed to catmaid::catmaid_fetch
#' @export
#' @rdname catmaid_skeletons_in_bbx
catmaid_skeletons_in_bbx <- function(bbx, min_nodes = 2, pid = 1, conn = NULL, ...){
  post_data = list()
  post_data["minx"] = bbx[1,1]
  post_data["miny"] = bbx[1,2]
  post_data["minz"] = bbx[1,3]
  post_data["maxx"] = bbx[2,1]
  post_data["maxy"] = bbx[2,2]
  post_data["maxz"] = bbx[2,3]
  post_data["min_nodes"] = min_nodes
  path = sprintf("/%d/skeletons/in-bounding-box", pid)
  res = catmaid::catmaid_fetch(path, body = post_data, include_headers = F,
                               simplifyVector = T, conn = conn, ...)
  res
}

#' Try to detect whether a neuron your are about to import into CATMAID might cause a duplication in that CATMAID instance
#'
#' @description  Programmatically search for skeleton IDs pertaining to neurons within a search volume defined by a bounding box.
#' @param neuron the neuron object you are thinking about uploading
#' @param skid the skeleton ID that corresponds to neuron
#' @param tolerance how many potentially duplicated nodes you will tolerate. If NULL, skeleton IDs for potentially duplicated neurons are returned.
#' @param duplication.range.nm determines the size of the bounding box around each node in neuron to search for a duplicated. Defaults to 10 nm
#' @param downsample if downsample > 1, a sample of number.of.points/downsample is taken from the neuron, and only these points are used to assess duplication. Speeds things up
#' @param fafbseg whether or not to use fafbseg::read_brainmaps_meshes on the TODO tag locations and restrict possible duplication to skeletons within the search range and within the cognate auto-segmented volume
#' @param pid project id. Defaults to 1
#' @param conn CATMAID connection object, see ?catmaid::catmaid_login for details
#' @param ... methods passed to catmaid::catmaid_fetch
#' @return If tolerance is NULL, then a list, with entries duplicated.nodes (a TRUE or FALSE for ever node in neuron, TRUE is there is another skeleton within duplication.range.nm)
#' and overlapping.skids, a list of skeleton IDs for potentially overlapping neurons. If tolerance is a numeric value between 0 and 1, TRUE or FALSE is returned.
#' @export
#' @rdname catmaid_duplicated
catmaid_duplicated <- function(neuron, skid = 0, tolerance = NULL, duplication.range.nm = 20, downsample = 1,  fafbseg = FALSE, pid = 1, conn = NULL, ...){
  duplicated = c()
  skids = list()
  if(!length(neuron$d)|nrow(neuron$d)==1){
    NULL
  }else{
    down = min(nrow(neuron$d), downsample)
    points = neuron$d[sample(1:nrow(neuron$d),ceiling(nrow(neuron$d)/down)),]
    pb <- utils::txtProgressBar(min = 0, max = nrow(points), style = 3)
    for(i in 1:nrow(points)){
      bbx = rbind(nat::xyzmatrix(points[i,])-duplication.range.nm, nat::xyzmatrix(points[i,])+duplication.range.nm)
      skids.bbx = catmaid_skeletons_in_bbx(bbx=bbx, min_nodes = 2, pid = pid, conn = conn, ...)
      skids.bbx = setdiff(skids.bbx,as.integer(skid))
      duplicated = c(duplicated,length(skids.bbx)>0)
      skids[[i]] = unique(skids.bbx)
      utils::setTxtProgressBar(pb, i)
    }
    close(pb)
    if(fafbseg&!is.null(unlist(skids))){
      if(!requireNamespace('fafbseg', quietly = TRUE))
        stop("Please install suggested fafbseg package")
      message("Checking whether potential duplicates are within the same FAFB volumetric auto-traced segments")
      similar.skids = unique(unlist(skids))
      similars = read.neurons.catmaid(similar.skids, OmitFailures = TRUE, pid = pid, conn = conn, ...)
      seg = tryCatch(fafbseg::brainmaps_xyz2id(points),error=function(e)NULL)
      seg = seg[seg!=0]
      s = tryCatch(fafbseg::find_merged_segments(seg),error=function(e)seg)
      s = s[s!=0]
      vol = tryCatch(suppressMessages(fafbseg::read_brainmaps_meshes(s)), error = function(e) message("warning: Google brainmaps read error"))
      in.vol = tryCatch(sapply(similars,function(n) sum(nat::pointsinside(nat::xyzmatrix(n),surf=vol))>0),error = function(e) rep(TRUE,length(similar.skids)))
      similar.skids = similar.skids[in.vol]
      skids = lapply(skids,function(s) s[s%in%similar.skids])
      duplicated = sapply(skids, function(s) length(s)>0)
    }
    if(is.null(tolerance)){
      list(duplicated.nodes = duplicated, overlapping.skids = skids)
    }else{
      calc = (sum(duplicated)/length(duplicated))
      ifelse(length(calc)>0,calc,0) > tolerance
    }
  }
}

#' Function used to delete single neurons from CATMAID based on a given skeleton ID
#'
#' @description  Delete a single neuron from a given CATMAID instance.
#' Deletes a neuron if and only if two things are the case: 1. The user
#' owns all treenodes of the skeleton modelling the neuron in question and
#' 2. The neuron is not annotated by other users.
#' Use with extreme caution as you may be significantly affecting others' work.
#' You will first be shown the neuron you want to delete and who has worked on it, and then asked whether or not you want to continue.
#' @param skid the skeleton ID for a single neuron you want to delete
#' @param skids if multiple neurons are to be deleted interactively, skids can be a vector of skeleton IDs, name or an annotation, anything that can be read by catmaid::catmaid_skids
#' @param connector_ids CATMAID connector IDs to be deleted, as long as they do not connect to any skeletons
#' @param delete_connectors if TRUE, connectors attached to the deleted skeleton, and no other skeleton (even if it is a single node) are also deleted
#' @param plot whether or not the plot the neuron you are considering deleting, before deciding to end it
#' @param brain the brain you want to plot alongside the neuron you are considering deleting.
#' @param max.nodes the maximum number of nodes a neuron can have, and still be successfully deleted. Helps prevent the accidental deletion of large neurons
#' @param control if TRUE (default) you will be asked at each stage if you wish to continue. Helps prevent accidental deletions. If FALSE, then will delete without asking.
#' Really should only be set to FALSE if you are deleting from your own CATMAID instance
#' @param pid project id. Defaults to 1
#' @param conn CATMAID connection object, see ?catmaid::catmaid_login for details
#' @param ... methods passed to catmaid::catmaid_fetch
#' @export
#' @rdname catmaid_delete
catmaid_delete_neuron <- function(skid,
                                  delete_connectors = TRUE,
                                  plot = TRUE, brain = NULL,
                                  max.nodes = 100, control = TRUE,
                                  pid = 1, conn = NULL, ...){
  if(length(skid)>1){
    stop("length(skid)>1 - You may only attempt to delete one neuron at a time.")
  }
  message("You are about to delete a NEURON from a CATMAID instance.
          This may affect people's hard work.
          You will first be shown the neuron you want to delete and who has worked on it,
          and then asked whether or not you want to continue.")
  if(plot){
    progress = readline(paste0("Do you want to continue with: ", skid," in CATMAID instance ", catmaid_get_server(conn), " y=yes, n=no : "))
  }else{
    progress = "y"
  }
  if(progress=="y"){
    nid = catmaid_get_neuronid(skids = skid, pid = pid, conn = conn, ...)
    neuron = tryCatch(catmaid::read.neurons.catmaid(skid, OmitFailures = TRUE, pid = pid, conn = conn, ...), error = function(e) "ERROR")
    if(neuron=="ERROR"){
      warning("Neuron with skeleton ID ", skid, " could not be read with pid ", pid, " in ", catmaid_get_server(conn))
    }else{
      connector_ids = unique(catmaid::connectors(neuron)$connector_id)
      xyz = nat::xyzmatrix(neuron)
      meta = summary(neuron)
      if(meta$nodes>max.nodes){
        stop("The neuron flagged for deletion has ", meta$nodes,
             " which is greater than the given argument max.nodes, ", max.nodes, ". Aborting.")
      }
      catmaid_url = paste0(catmaid_get_server(conn), "?pid=",pid)
      catmaid_url = paste0(catmaid_url, "&zp=", xyz[1,"Z"])
      catmaid_url = paste0(catmaid_url, "&yp=", xyz[1,"Y"])
      catmaid_url = paste0(catmaid_url, "&xp=", xyz[1,"X"])
      catmaid_url = paste0(catmaid_url, "&tool=tracingtool")
      catmaid_url = paste0(catmaid_url, "&active_skeleton_id=", skid)
      catmaid_url = paste0(catmaid_url, "&sid0=5&s0=0")
      ### Plot ###
      if(plot){
        rgl::clear3d()
        if(!is.null(brain)){
          rgl::plot3d(brain,alpha=0.1,col="grey")
        }
        rgl::plot3d(neuron,lwd=3,soma=T,WithConnectors = TRUE, col ="black")
      }
      ### Decide ###
      print(cbind(neuron[,], meta))
      print(paste0("See neuron in CATMAID at: ", catmaid_url))
      decide = "n"
      if(control){
        decide = readline(paste0("Are you SURE you want to delete: ", skid," in CATMAID instance ", catmaid_get_server(conn), "? y=yes, n=no : "))
      }else{
        decide = "y"
      }
      if(decide=="y"){
        path = sprintf("/%d/neuron/%d/delete", pid, nid)
        res = catmaid_fetch(path, body = NULL, include_headers = F,
                            conn = conn, ...)
        if(!is.null(res$success)){
          message(res$success)
          if(delete_connectors&!is.null(connector_ids)){
            message("Now deleting free connectors")
            catmaid_delete_connectors(connector_ids = connector_ids, pid = pid, conn = conn, ...)
          }
        }else{
          message(res$error)
        }
        message("This activity has been logged.")
      }
    }
  }
}
#' @export
#' @rdname catmaid_delete
catmaid_delete_connectors <- function(connector_ids, pid = 1, conn = NULL, ...){
  delete = c()
  for(i in 1:length(connector_ids)){
    post_data = list()
    post_data["connector_ids[0]"] = as.numeric(connector_ids[i])
    path = sprintf("/%d/connector/skeletons", pid)
    res = catmaid::catmaid_fetch(path, body = post_data, include_headers = F,
                                 simplifyVector = T, conn = conn,...)
    invisible(catmaid:::catmaid_error_check(res2))
    del = ifelse(length(res),FALSE,TRUE)
    delete = c(delete, del)
  }
  connector_ids_delete = connector_ids[delete]
  if(length(connector_ids_delete)){
    for(connector_id_delete in connector_ids_delete){
      ctime = catmaid_node_time(id=connector_id_delete, time = "edition_time", pid=pid, conn=conn, ...)
      post_data = list()
      post_data["connector_id"] = connector_id_delete
      post_data["state"] =  sprintf('{"edition_time":"%s","c_links": []}',ctime)
      path = sprintf("/%d/connector/delete", pid)
      res = catmaid::catmaid_fetch(path, body = post_data, include_headers = F,
                                   simplifyVector = T, conn = conn,...)
      invisible(catmaid:::catmaid_error_check(res))
      if(is.null(res$error)){
        message(res$message)
      }else{
        warning(res$error)
      }
    }
  }
}
#' @export
#' @rdname catmaid_delete
catmaid_delete_neurons <- function(skids,
                                   delete_connectors = TRUE,
                                   plot = TRUE, brain = NULL,
                                   max.nodes = 100, control = TRUE,
                                   pid = 1, conn = NULL, ...){
  delete.skids = catmaid::catmaid_skids(skids, pid = pid, conn = conn, ...)
  deletions = sapply(delete.skids,catmaid_delete_neuron,delete_connectors=delete_connectors,
                     plot=plot,brain=brain,max.nodes=max.nodes,control=control,pid=pid,conn=conn,...)
}

### moved to rcatmaid ###
#' Get the CATMAID neuron ID that corresponds to the skeleton ID
#'
#' @description Retrieve the neuron IDs for given skeleton IDs. This is typically the skeleton ID + 1, and is often, but not always accurately, kept by CATMAID tracers in the name of a neuron.
#' @param skids a vector of skeleton IDs or argument applicable to catmaid::catmaid_get_neuronid
#' @param pid project id. Defaults to 1
#' @param conn CATMAID connection object, see ?catmaid::catmaid_login for details
#' @param ... methods passed to catmaid::catmaid_fetch
#' @export
#' @rdname catmaid_get_neuronid
catmaid_get_neuronid <- function(skids, pid = 1, conn = NULL, ...){
  skids = catmaid_skids(skids, conn = conn, pid = pid, ...)
  if (any(duplicated(skids))) {
    uskids = unique(skids)
    unids = catmaid_get_neuronid(uskids, pid = pid, conn = conn,...)
    res = unids[match(skids, uskids)]
    return(res)
  }
  skids[is.na(skids)] = -1L
  res = lapply(skids,function(skid)
    catmaid::catmaid_fetch(sprintf("/%d/skeleton/%s/neuronname", pid, skid), body = NULL, include_headers = F,
                           conn = conn, ...)$neuronid)
  res = sapply(res,function(r) ifelse(is.null(r),NA,r))
  names(res) = skids
  res
}

### added to rcatmaid ###
#' Lock or unlock a CATMAID neuron reconstruction
#'
#' @description  Lock or unlock a CATMAID neuron reconstruction by adding or removing a 'locked' annotation to a set of skeleton IDs (skids). A locked neuron cannot be edited until it is unlocked.
#' @param skids the skeleton IDs neurons you wish to lock / unlock
#' @param pid project id. Defaults to 1
#' @param conn CATMAID connection object, see ?catmaid::catmaid_login for details
#' @param ... methods passed to catmaid::catmaid_fetch
#' @export
#' @rdname catmaid_lock_neurons
catmaid_lock_neurons <- function(skids, pid = 1, conn = NULL, ...){
  skids = catmaid::catmaid_skids(skids, pid=pid,conn=conn,...)
  catmaid::catmaid_set_annotations_for_skeletons(skids, annotations = "locked", pid = pid,
                                                 conn = conn, ...)
}
#' @export
#' @rdname catmaid_lock_neurons
catmaid_unlock_neurons <- function(skids, pid = 1, conn = NULL, ...){
  skids = catmaid::catmaid_skids(skids, pid=pid,conn=conn,...)
  catmaid::catmaid_remove_annotations_for_skeletons(skids, annotations = "locked", pid = pid,
                                                    conn = conn, ...)
}

# A helper function, not exported
catmaid_convert_time <- function(utc){
  t = format(as.POSIXlt(utc,tz="GMT",origin="1970-01-01"), "%Y-%m-%d %H:%M:%OS3")
  s = unlist(strsplit(t," "))
  t = paste0(s[1],"T",s[2],"Z")
}

#' Interactively upload neuron(s) to CATMAID
#'
#' @description  Uploads neurons to CATMAID, names them and annotates them, from the environment specified with \code{conn} to that specified by \code{conn2}.
#' Please use with caution, as you could be heavily adding to a live tracing environment. When neurons are shown, potential join sites / locations of join tags are shown as spheres.
#' @param x either skeleton IDs in the environment specified by conn2 (by default, the v14-seg CATMAID instance), or a neuronlist object to upload to the CATMAID instance specified when you use catmaid::catmaid_login(), if conn is NULL, else specified by conn$server
#' @param tolerance how many potentially duplicated nodes you will tolerate
#' @param name whatever you want to name your uploaded neurons. If a single character, then it will be added to all uploaded neurons. Else, can be a character vector the same length as swc.
#' @param annotations a character vector of annotations, to be added to all of the uploaded neurons
#' @param avoid a character vector. If a neuron contains an annotation specified by avoid, it will not be considered for upload
#' @param include.tags whether of not to transfer each neuron's tags to its newly uploaded cognate
#' @param include.connectors whether of not to transfer each neuron's connectors to its newly uploaded cognate
#' @param join whether or not to attempt to join each uploaded neuron to neurons within the new CATMAID instance, based on the placement of join.tags specified using the next argument
#' @param join.tag a single character specifying a tag that has been used to signify a potential merge point
#' @param lock if TRUE, neurons in the CATMAID instance specified by conn2 (i.e. the environment from which you are uploading) will be 'locked' so other users cannot edit them (via addition of a 'locked' annotation)
#' @param min_nodes the minimum number of nodes a potential merger skeleton needs to have
#' @param search.range.nm the maximum distance from which to search from the TODO point tag to find potential mergers
#' @param duplication.range.nm the radius around each node of a skeleton you are looking to upload, in which to search for other CATMAID skeletons that might represent a duplication of the same neuron
#' @param include.potential.duplicates whether or not to include skeletons from the instance specified by conn2, that have the annotation "duplicated"
#' @param fafbseg whether or not to use fafbseg::read_brainmaps_meshes on the TODO tag locations and restrict possible merges to neurons within the search range and within the cognate auto-segmented volume
#' @param downsample if downsample > 1, a sample of number.of.points/downsample is taken from the neuron, and only these points are used to assess duplication. Speeds things up
#' @param return.uploaded.skids whether or not to returns the skeleton IDs of uploaded neurons. If FALSE, nothing is returned
#' @param brain the brain to plot while visualising potential mergers using rgl. Defaults to NULL, no brain plotted.
#' @param pid project id for conn. Defaults to 1
#' @param pid2 project id for conn2. Defaults to 1
#' @param conn CATMAID connection object, see ?catmaid::catmaid_login for details
#' @param conn2 CATMAID connection object, see ?catmaid::catmaid_login for details
#' @param ... methods passed to catmaid::catmaid_fetch
#' @examples
#' \dontrun{
#' # This function, first, gets the neurons we want from the v14-seg CATMAID instance
#' # Then it checks that we have not already uploaded to v14-seg using the annotations you specify with the avoid argument
#' # Then it will seek to upload them in a controlled way, that gives us options to have their tags, connectors, and make joins, as well as trying to check for possible duplication.
#' # Be aware that the interactive join will often suggest the same neuron multiple times at different join sites.
#' # The join sites are indicated by spheres - you may need to zoom in in order to see them.
#' # If lock = TRUE, the neurons we just uploaded will be locked, to lessen the chance someone else will connect stuff to them and want to upload them AGAIN later...
#' uploaded = catmaid_controlled_upload(x = "name:ASB Tester", join = TRUE, name = "ASB Tester from v14-seg", duplication.range.nm = 10,
#' search.range.nm = 1000, annotations = "ASB Test v14-seg Upload", brain = elmr::FAFB14)
#' # Note that CATMAID links will also be supplied, so you can inspect a merge site in CATMAID. If you join in CATMAID, do not join in the interactive window, ad this will throw an errror, just keep hitting 'n' for no, until all options are exhausted.
#' # Oops, did you make a mistake in uploading this neuron?
#' catmaid_delete_neurons("annotation:ASB Tester from v14-seg")
#' # Be careful wen deleting, especially if you have merged your fragment during the interactive join.
#' # Phew.
#' }
#' @export
#' @rdname catmaid_controlled_upload
catmaid_controlled_upload <- function(x, tolerance = 0.15, name = "v14-seg neuron upload",
                                      annotations = "v14-seg upload", avoid = "v14",
                                      avoid.join= NULL, join.only = NULL,
                                      include.potential.duplicates = FALSE,
                                      include.tags = TRUE, include.connectors = TRUE,
                                      search.range.nm = 1000, join = FALSE, join.tag = "TODO",
                                      lock = TRUE, fafbseg = FALSE, downsample = 2, min_nodes = 2,
                                      brain = NULL, return.uploaded.skids = TRUE,duplication.range.nm=20,
                                      pid = 1, conn = NULL, pid2 = 1, conn2 = fafb_seg_conn(),  ...){
  if(!nat::is.neuronlist(x)){
    message("Reading neurons from ", catmaid_get_server(conn2))
    neurons = catmaid::read.neurons.catmaid(x, OmitFailures = TRUE, pid=pid2,conn=conn2, ...)
    anns = catmaid::catmaid_get_annotations_for_skeletons(x,pid=pid2,conn=conn2,...)
    if("annotation"%in%colnames(anns)){
      if(include.potential.duplicates){
        avoiding  = avoid
      }else{
        avoiding = c(avoid,"duplicated")
      }
      already.there = subset(anns,annotation%in%avoiding)$skid
      neurons = neurons[setdiff(names(neurons),already.there)]
      if(length(already.there)){
        message(length(already.there), " neurons have an annotation that indicates that they should not be uploaded: ", avoid)
        if(length(neurons)==length(already.there)){
          stop("No neurons to upload, you can try setting the argument include.potential.duplicates to TRUE")
        }
      }
    }
  }else{
    neurons = x
  }
  if(is.character(name)){
    if(length(name)==1&length(neurons)>1){
      names = paste0(name,"_",seq_along(neurons))
    } else if (length(name)!=length(neurons)){
      stop(paste0('The names argument mut be a vector or length 1, to be applied to all neurons
                  or a vector the same length as the number of neurons specified by x'))
    }else{
      names = name
    }
  }else{
    stop("name must be a character vector")
  }
  if(lock){
    avoid = unique(c(avoid,"locked"))
  }
  uploaded.new = c()
  uploaded.old = c()
  message("Considering upload to ", catmaid_get_server(conn))
  nat::nopen3d()
  for(i in 1:length(neurons)){
    neuron = neurons[[i]]
    old.skid = neurons[i,"skid"]
    if(length(name)==1){
      nam = names
    }else{
      nam = names[i]
    }
    rgl::clear3d()
    if(!is.null(brain)){
      rgl::plot3d(brain,alpha=0.1,col="grey")
    }
    rgl::plot3d(neuron, lwd = 1, WithConnectors = TRUE)
    tag.point  = catmaid_get_tag(neuron, tag = join.tag)
    if(length(tag.point)){
      rgl::spheres3d(nat::xyzmatrix(tag.point),col="orange",alpha=0.5,radius=search.range.nm)
    }
    message("Assessing whether upload of this neuron ",i, " of ", length(neurons), " may cause a duplication:" )
    print(neurons[i,])
    dupe = catmaid_duplicated(neuron, tolerance = NULL, duplication.range.nm = duplication.range.nm, downsample = downsample, fafbseg = fafbseg, pid = pid, conn = conn, ...)
    calc = (sum(dupe$duplicated.nodes)/length(dupe$duplicated.nodes))
    calc = ifelse(length(calc)>0,calc,0)
    if(calc>0){
      t = table(unlist(dupe$overlapping.skids))
      t = names(t)[order(t,decreasing = TRUE)]
      t = t[1:min(3,length(t))]; t = t[!is.na(t)]
      similar = catmaid::read.neurons.catmaid(t, OmitFailures = TRUE, pid = pid, conn = conn, ...)
      rgl::plot3d(similar, col=c("darkgrey","grey","lightgrey")[1:length(similar)], lwd=4, WithConnectors = TRUE)
      message("most overlapping neurons:")
      print(similar[,])
    }
    dupe = calc > tolerance
    message("The neuron flagged for upload seems to have ", calc*100, "% of its nodes may already be in the targetted CATMAID instance")
    progress = "s"
    while(!progress%in%c("y","n","a")){
      progress = readline("Do you want to upload this neuron? y=yes, n=no, a=no + annotate as duplicated: ")
      if(progress=="y"){
        progress = readline("Sure? y=yes, n=no, a=no + annotate as duplicated: ")
      }
      if(progress=="a"){
        catmaid::catmaid_set_annotations_for_skeletons(skids = old.skid, annotations = "duplicated", pid = pid2, conn = conn2, ...)
      }else if (dupe){
        message("Neuron ", i, " with skid ", old.skid, " appears to already exist in the CATMAID instance to which you are seeking to upload.
                Upload for neuron ", i, " aborted (tolerance:",tolerance,").")
      }else if(progress=="y"){
        if(calc>0){
          nat::npop3d()
        }
        message("Uploading neuron ", i)
        new.skid = catmaid_upload_neurons(swc=neuron,name=names[i],annotations = c(annotations),
                                          include.tags=include.tags,include.connectors=include.connectors,
                                          return.new.skids = TRUE,
                                          conn = conn, pid = pid, max.upload = 1, ...)
        new.neuron = read.neurons.catmaid(new.skid, OmitFailures = TRUE, conn=conn, pid=pid, ...)
        message("Upload successful, neuron ", new.skid, " created, named: ", new.neuron[,"name"])
        catmaid::catmaid_set_annotations_for_skeletons(skids = old.skid, annotations = avoid, pid = pid2, conn = conn2, ...)
        uploaded.new = c(uploaded.new,new.skid)
        uploaded.old = c(uploaded.old,old.skid)
        if(join){
          message("Finding tagged potential join points ...")
          TODO = tryCatch(catmaid_get_tag(x = new.neuron, tag = join.tag, url = FALSE),
                          error = function(e) message("Could not find merge related tags, no joining to be done"))
          if(length(TODO)){
            possible.merges = tryCatch(catmaid_find_likely_merge(TODO = TODO, pid=pid, fafbseg = fafbseg, conn = conn,
                                                                 min_nodes = min_nodes, search.range.nm = search.range.nm, ...),
                                       error = function(e) message("Error finding join sites, aborting join"))
            if(!is.null(avoid.join)|!is.null(join.only)){
              upstream.annotations = catmaid::catmaid_get_annotations_for_skeletons(unique(possible.merges$upstream.skid), pid = pid, conn = conn, ...)
              if(!is.null(avoid.join)){
                message("Removing any join taget with annotations specified by avoid.join")
                acceptable.skids = setdiff(unique(possible.merges$upstream.skid),unique(subset(upstream.annotations,annotation%in%avoid.join)$skid))
                possible.merges = subset(possible.merges,upstream.skid%in%acceptable.skids)
              }
              if(!is.null(join.only)){
                message("Choosing only join tagets with annotations specified by join.only")
                acceptable.skids = intersect(unique(possible.merges$upstream.skid),unique(subset(upstream.annotations,annotation%in%join.only)$skid))
                possible.merges = subset(possible.merges,upstream.skid%in%acceptable.skids)
              }
            }
            if(length(possible.merges)){possible.merges = subset(possible.merges,upstream.skid!=new.skid)}
            if(length(possible.merges)){
              message("Choose join sites interactively: ")
              tryCatch(catmaid_interactive_join(possible.merges=possible.merges, downstream.neurons = new.neuron, brain = brain, pid = pid, conn = conn, ...),
                       error = function(e) message("Error making join, aborting join"))
            }else{
              message("No potential join sites found for new neuron ", new.skid)
              next
            }
          }else{
            message("No potential join sites found for new neuron ", new.skid)
            next
          }
        }
      }
  }
  }
  if(return.uploaded.skids){
    list(uploaded.skids = uploaded.new, downloaded.skids = uploaded.old)
  }
  }

# Hidden function, for efficiency
catmaid_uncontrolled_upload <- function(x, tolerance = 0, name = "v14-seg neuron upload",
                                        annotations = "v14-seg upload", avoid = "v14",
                                        avoid.join = NULL, join.only = NULL,
                                        include.tags = TRUE, include.connectors = TRUE,
                                        search.range.nm = 1000, duplication.range.nm = 10, join = TRUE,
                                        join.tag = "TODO", include.potential.duplicates = TRUE,
                                        lock = TRUE, fafbseg = TRUE, downsample = 2, min_nodes = 2,
                                        return.uploaded.skids = TRUE,
                                        pid = 1, conn = NULL, pid2 = 1, conn2 = fafb_seg_conn(),  ...){
  if(!nat::is.neuronlist(x)){
    message("Reading neurons from ", catmaid_get_server(conn2))
    neurons = catmaid::read.neurons.catmaid(x, OmitFailures = TRUE, pid=pid2,conn=conn2, ...)
    anns = catmaid::catmaid_get_annotations_for_skeletons(x,pid=pid2,conn=conn2,...)
    if("annotation"%in%colnames(anns)){
      if(include.potential.duplicates){
        avoiding  = avoid
      }else{
        avoiding = c(avoid,"duplicated")
      }
      already.there = subset(anns,annotation%in%avoiding)$skid
      neurons = neurons[setdiff(names(neurons),already.there)]
      if(length(already.there)){
        message(length(already.there), " neurons have an annotation that indicates that they should not be uploaded: ", avoid)
        if(length(neurons)==length(already.there)){
          stop("No neurons to upload, you can try setting the argument include.potential.duplicates to TRUE")
        }
      }
    }
  }else{
    neurons = x
  }
  if(is.character(name)){
    if(length(name)==1&length(neurons)>1){
      names = paste0(name,"_",seq_along(neurons))
    } else if (length(name)!=length(neurons)){
      stop(paste0('The names argument mut be a vector or length 1, to be applied to all neurons
                  or a vector the same length as the number of neurons specified by x'))
    }else{
      names = name
    }
  }else{
    stop("name must be a character vector")
  }
  if(lock){
    avoid = unique(c(avoid,"locked"))
  }
  uploaded.new = c()
  uploaded.old = c()
  message("Considering upload to ", catmaid_get_server(conn))
  for(i in 1:length(neurons)){
    neuron = neurons[[i]]
    old.skid = neurons[i,"skid"]
    message("Assessing whether upload of this neuron ",i, " of ", length(neurons), " may cause a duplication:" )
    print(neurons[i,])
    dupe = catmaid_duplicated(neuron, tolerance = NULL, duplication.range.nm = duplication.range.nm, downsample = downsample, fafbseg = fafbseg, pid = pid, conn = conn, ...)
    calc = (sum(dupe$duplicated.nodes)/length(dupe$duplicated.nodes))
    calc = ifelse(length(calc)>0,calc,0)
    dupe = calc > tolerance
    message("The neuron flagged for upload seems to have ", calc*100, "% of its nodes already in the targetted CATMAID instance")
    if(dupe){
      message("Neuron ", i, " with skid ", old.skid, " appears to already exist in the CATMAID instance to which you are seeking to upload.
              Upload for neuron ", i, " aborted (tolerance: ",tolerance,").")
    }else{
      message("Uploading neuron ", i)
      new.skid = catmaid_upload_neurons(swc=neuron,name=names[i], annotations = c(annotations),
                                        include.tags=include.tags,include.connectors=include.connectors,
                                        return.new.skids = TRUE,
                                        conn = conn, pid = pid, max.upload = 1, ...)
      new.neuron = read.neurons.catmaid(new.skid, OmitFailures = TRUE, conn=conn, pid=pid, ...)
      message("Upload successful, neuron ", new.skid, " created, named: ", new.neuron[,"name"])
      catmaid::catmaid_set_annotations_for_skeletons(skids = old.skid, annotations = avoid, pid = pid2, conn = conn2, ...)
      uploaded.new = c(uploaded.new,new.skid)
      uploaded.old = c(uploaded.old,old.skid)
      if(join){
        message("Finding tagged potential join points ...")
        TODO = tryCatch(catmaid_get_tag(x = new.neuron, tag = join.tag, url = FALSE),
                          error = function(e) message("Could not find merge related tags, no joining to be done"))
        if(length(TODO)){
          possible.merges = tryCatch(catmaid_find_likely_merge(TODO = TODO, pid=pid, fafbseg = fafbseg, conn = conn,
                                                               min_nodes = min_nodes, search.range.nm = search.range.nm, ...),
                                     error = function(e) message("Error finding join sites, aborting join"))
          if(!is.null(avoid.join)|!is.null(join.only)){
            upstream.annotations = catmaid::catmaid_get_annotations_for_skeletons(unique(possible.merges$upstream.skid), pid = pid, conn = conn, ...)
            if(!is.null(avoid.join)){
              message("Removing any join target with annotations specified by avoid.join")
              acceptable.skids = setdiff(unique(possible.merges$upstream.skid),unique(subset(upstream.annotations,annotation%in%avoid.join)$skid))
              possible.merges = subset(possible.merges,upstream.skid%in%acceptable.skids)
            }
            if(!is.null(join.only)){
              message("Choosing only join targets with annotations specified by join.only")
              acceptable.skids = intersect(unique(possible.merges$upstream.skid),unique(subset(upstream.annotations,annotation%in%join.only)$skid))
              possible.merges = subset(possible.merges,upstream.skid%in%acceptable.skids)
            }
          }
          if(length(possible.merges)){possible.merges = subset(possible.merges,upstream.skid!=new.skid)}
          if(length(possible.merges)){
            message("making joins")
            possible.merges$downstream.nodes  = catmaid::catmaid_get_node_count(possible.merges$downstream.skid)
            possible.merges$upstream.nodes  = catmaid::catmaid_get_node_count(possible.merges$upstream.skid)
            multiple.joins = FALSE # First merge retains name and skid of the manual neuron, rather than the larger
            last.merge.size = 1e6
            for(todo in unique(possible.merges$downstream.node)){
              TODO.possible = subset(possible.merges,downstream.node==todo)
              downstream.node = catmaid::catmaid_get_treenodes_detail(tnids = todo, pid = pid, conn = conn, ...)
              merger = catmaid::catmaid_get_treenodes_detail(tnids = TODO.possible[1,"upstream.node"], pid = pid, conn = conn, ...)
              merger.neuron = catmaid::catmaid_get_neuronnames(as.character(merger$skid), pid = pid, conn = conn, ...)
              message("Merging to:  ",paste(merger.neuron,collapse = " "))
              catmaid_url = paste0(catmaid_get_server(conn), "?pid=",pid)
              catmaid_url = paste0(catmaid_url, "&zp=", TODO.possible[1,"Z"])
              catmaid_url = paste0(catmaid_url, "&yp=", TODO.possible[1,"Y"])
              catmaid_url = paste0(catmaid_url, "&xp=", TODO.possible[1,"X"])
              catmaid_url = paste0(catmaid_url, "&tool=tracingtool")
              catmaid_url = paste0(catmaid_url, "&active_skeleton_id=", new.skid)
              catmaid_url = paste0(catmaid_url, "&sid0=5&s0=0")
              message("See merge site in CATMAID: ", catmaid_url)
              # find out which skeleton is larger
              if(multiple.joins){
                message("multiple joins for the same uploaded fragment being made")
                if(last.merge.size>TODO.possible[1,"upstream.nodes"]){
                  from_treenode_id = TODO.possible[1,"upstream.node"]
                  to_treenode_id = TODO.possible[1,"downstream.node"]
                }else{
                  from_treenode_id = TODO.possible[1,"downstream.node"]
                  to_treenode_id = TODO.possible[1,"upstream.node"]
                }
                last.merge.size = catmaid::catmaid_get_node_count(as.character(merger$skid)) + unique(TODO.possible$downstream.nodes)
                catmaid_join_skeletons(from_treenode_id = from_treenode_id, to_treenode_id = to_treenode_id, pid = pid, conn = conn, ...)
              }else{
                last.merge.size = catmaid::catmaid_get_node_count(as.character(merger$skid)) + unique(TODO.possible$downstream.nodes)
                catmaid_join_skeletons(from_treenode_id = TODO.possible[1,"downstream.node"], to_treenode_id = TODO.possible[1,"upstream.node"], pid = pid, conn = conn, ...)
              }
              multiple.joins = TRUE
            }
          }else{
            message("No potential join sites found for new neuron ", new.skid)
            next
          }
        }else{
          message("No potential join sites found for new neuron ", new.skid)
          next
        }
      }
    }
  }
  if(return.uploaded.skids){
    list(uploaded.skids = uploaded.new, downloaded.skids = uploaded.old)
  }
  }

#' Connect to a local CATMAID server
#'
#' @description connect to a local CATMAID server running on you machine using Docker (see ?catmaid::catmaid_login, and https://catmaid.readthedocs.io/en/stable/docker.html)
#' @export
#' @rdname local_conn
local_conn <- function(){
  catmaid::catmaid_login(server = "http://localhost:8000/",
                         token = "5c93cd0d5a75427aac7d1e39f3deb0cd59de19e7",
                         Cache = FALSE)
}

### moved to rcatmaid ###
#' Get CATMAID server
#'
#' @description shows the URL for a connection object (see ?catmaid::catmaid_login)
#' @param conn CATMAID connection object, see ?catmaid::catmaid_login for details
#' @param ... methods passed to catmaid::catmaid_login
#' @export
#' @rdname catmaid_get_server
catmaid_get_server<-function(conn=NULL,...){
  if(is.null(conn)){
    conn = catmaid::catmaid_login()
  }
  conn$server
}

# Hidden joining function
catmaid_batch_join <- function(x, fafbseg = TRUE, join.tag = "TODO", max_size = 2000,
                               min_nodes = 2, search.range.nm = 100,
                               pid = pid, conn = conn, ...){
  if(is.null(conn)){
    conn = catmaid::catmaid_login()
  }
  if(!nat::is.neuronlist(x)){
    message("Reading neurons from ", catmaid_get_server(conn))
    neurons = catmaid::read.neurons.catmaid(x, OmitFailures = TRUE, pid=pid,conn=conn, ...)
    sum = summary(neurons)
    neurons = neurons[sum$nodes<max_size]
  }else{
    neurons = x
  }
  message("Considering making joins in ", catmaid_get_server(conn), " for ", length(neurons), " neurons")
  for (i in 1:length(neurons)){
    neuron.skid = names(neurons)[i]
    neuron = neurons[i][[1]]
    message("Finding tagged potential join points ...")
    TODO = catmaid_get_tag(x = neuron, tag = join.tag, url = FALSE)
    if(length(TODO)){
      message(nrow(TODO), " nodes tagged with ", join.tag, " for: ")
      print(neurons[i,])
      possible.merges = tryCatch(catmaid_find_likely_merge(TODO = TODO, pid=pid, fafbseg = fafbseg, conn = conn,
                                                           min_nodes = min_nodes, search.range.nm = search.range.nm, ...),
                                 error = function(e) message("Error finding join sites, aborting join"))
      if(length(possible.merges)){possible.merges = subset(possible.merges,upstream.skid!=neuron.skid)}
      if(length(possible.merges)){
        message("making joins for")
        print(neurons[i,])
        for(todo in unique(possible.merges$downstream.node)){
          TODO.possible = subset(possible.merges,downstream.node==todo)
          downstream.node = catmaid::catmaid_get_treenodes_detail(tnids = todo, pid = pid, conn = conn, ...)
          merger = catmaid::catmaid_get_treenodes_detail(tnids = TODO.possible[1,"upstream.node"], pid = pid, conn = conn, ...)
          merger.neuron = catmaid::catmaid_get_neuronnames(as.character(merger$skid), pid = pid, conn = conn, ...)
          message("Merging to:  ",paste(merger.neuron,collapse = " "))
          catmaid_url = paste0(catmaid_get_server(conn), "?pid=",pid)
          catmaid_url = paste0(catmaid_url, "&zp=", TODO.possible[1,"Z"])
          catmaid_url = paste0(catmaid_url, "&yp=", TODO.possible[1,"Y"])
          catmaid_url = paste0(catmaid_url, "&xp=", TODO.possible[1,"X"])
          catmaid_url = paste0(catmaid_url, "&tool=tracingtool")
          catmaid_url = paste0(catmaid_url, "&active_skeleton_id=", new.skid)
          catmaid_url = paste0(catmaid_url, "&sid0=5&s0=0")
          message("See merge site in CATMAID: ", catmaid_url)
          tryCatch(catmaid_join_skeletons(from_treenode_id = TODO.possible[1,"downstream.node"], to_treenode_id = TODO.possible[1,"upstream.node"], pid = pid, conn = conn, ...),
                   error = function(e) message("Error making join (likely loop), join aborted"))
        }
      }else{
        message("No potential join sites found for neuron ", neuron.skid)
        next
      }
    }else{
      message("No potential join sites found for neuron ", neuron.skid)
      next
    }
  }
}

# Hidden joining function
catmaid_batch_join <- function(x, fafbseg = TRUE, join.tag = "TODO", max_size = 2000,
                               min_nodes = 2, search.range.nm = 100,
                               pid = pid, conn = conn, ...){
  if(is.null(conn)){
    conn = catmaid::catmaid_login()
  }
  if(!nat::is.neuronlist(x)){
    message("Reading neurons from ", catmaid_get_server(conn))
    neurons = catmaid::read.neurons.catmaid(x, OmitFailures = TRUE, pid=pid,conn=conn, ...)
    sum = summary(neurons)
    neurons = neurons[sum$nodes<max_size]
  }else{
    neurons = x
  }
  message("Considering making joins in ", catmaid_get_server(conn), " for ", length(neurons), " neurons")
  for (i in 1:length(neurons)){
    neuron.skid = names(neurons)[i]
    neuron = neurons[i][[1]]
    message("Finding tagged potential join points ...")
    TODO = catmaid_get_tag(x = neuron, tag = join.tag, url = FALSE)
    if(length(TODO)){
      message(nrow(TODO), " nodes tagged with ", join.tag, " for: ")
      print(neurons[i,])
      possible.merges = tryCatch(catmaid_find_likely_merge(TODO = TODO, pid=pid, fafbseg = fafbseg, conn = conn,
                                                           min_nodes = min_nodes, search.range.nm = search.range.nm, ...),
                                 error = function(e) message("Error finding join sites, aborting join"))
      if(length(possible.merges)){possible.merges = subset(possible.merges,upstream.skid!=neuron.skid)}
      if(length(possible.merges)){
        message("making joins for")
        print(neurons[i,])
        for(todo in unique(possible.merges$downstream.node)){
          TODO.possible = subset(possible.merges,downstream.node==todo)
          downstream.node = catmaid::catmaid_get_treenodes_detail(tnids = todo, pid = pid, conn = conn, ...)
          merger = catmaid::catmaid_get_treenodes_detail(tnids = TODO.possible[1,"upstream.node"], pid = pid, conn = conn, ...)
          merger.neuron = catmaid::catmaid_get_neuronnames(as.character(merger$skid), pid = pid, conn = conn, ...)
          message("Merging to:  ",paste(merger.neuron,collapse = " "))
          catmaid_url = paste0(catmaid_get_server(conn), "?pid=",pid)
          catmaid_url = paste0(catmaid_url, "&zp=", TODO.possible[1,"Z"])
          catmaid_url = paste0(catmaid_url, "&yp=", TODO.possible[1,"Y"])
          catmaid_url = paste0(catmaid_url, "&xp=", TODO.possible[1,"X"])
          catmaid_url = paste0(catmaid_url, "&tool=tracingtool")
          catmaid_url = paste0(catmaid_url, "&active_skeleton_id=", new.skid)
          catmaid_url = paste0(catmaid_url, "&sid0=5&s0=0")
          message("See merge site in CATMAID: ", catmaid_url)
          tryCatch(catmaid_join_skeletons(from_treenode_id = TODO.possible[1,"downstream.node"], to_treenode_id = TODO.possible[1,"upstream.node"], pid = pid, conn = conn, ...),
                   error = function(e) message("Error making join (likely loop), join aborted"))
        }
      }else{
        message("No potential join sites found for neuron ", neuron.skid)
        next
      }
    }else{
      message("No potential join sites found for neuron ", neuron.skid)
      next
    }
  }
}

#' Update radius information for tree nodes in a CATMAID instance
#'
#' @description  Update radius information for tree nodes in a CATMAID instance. A locked neuron cannot be edited until it is unlocked.
#' @param tnids the treenode ids to edit
#' @param radii a vector the same length as tnids, giving the new radius for each treenode id in that vector
#' @param pid project id. Defaults to 1
#' @param conn CATMAID connection object, see ?catmaid::catmaid_login for details
#' @param ... methods passed to catmaid::catmaid_fetch
#' @export
#' @rdname catmaid_update_radius
catmaid_update_radius <- function(tnids, radii, pid = 1, conn = NULL, ...){
  tnids = tnids[!is.na(radii)]
  radii = radii[!is.na(radii)]
  pb <- utils::txtProgressBar(min = 0, max = length(tnids), style = 3)
  for(i in 1:length(tnids)){
    post_data = list()
    post_data["radius"] = radii[i]
    tnid_time = catmaid_node_time(id=tnids[i], time = "edition_time", pid=pid, conn=conn, ...)
    post_data["state"] = sprintf('{"edition_time":"%s"}',tnid_time)
    path = sprintf("/%d/treenode/%d/radius", pid, tnids[i])
    res = catmaid::catmaid_fetch(path, body = post_data, include_headers = F,
                                 simplifyVector = T, conn = conn, ...)
    invisible(catmaid:::catmaid_error_check(res))
    utils::setTxtProgressBar(pb, i)
  }
  close(pb)
}


# uploaded1 = catmaid_uncontrolled_upload(x ="annotation:downstream of aBN1", tolerance = 0.8, name = "downstream of antennal BN1 KE_",
#                                       annotations = c("v14 upload", "downstream of aBN1"), avoid = "v14 upload", lock = FALSE,
#                                       include.tags = TRUE, include.connectors = FALSE, downsample = 10,
#                                       search.range.nm = 100, duplication.range.nm=100, join = FALSE, join.tag = "TODO",
#                                       fafbseg = FALSE, min_nodes = 2, return.uploaded.skids = TRUE,
#                                       pid2 = 1, conn2 = NULL, pid = 1, conn = fafb_seg_conn(),
#                                       include.potential.duplicates = FALSE, join.only = NULL, avoid.join = NULL)

#'
#' #' # Test these functions
# uploaded = catmaid_controlled_upload(x ="annotation:ASB downseg", tolerance = 0.8, name = "v14-seg neuron upload ASB",
#                                        annotations = c("v14-seg upload", "ASB upseg"), avoid = "v14", lock = TRUE,
#                                        include.tags = TRUE, include.connectors = FALSE, downsample = 2, include.potential.duplicates=TRUE,
#                                        search.range.nm = 500, duplication.range.nm=100, join = TRUE, join.tag = "TODO",
#                                        fafbseg = TRUE, min_nodes = 2, return.uploaded.skids = TRUE,
#                                        pid = 1, conn = NULL, pid2 = 1, conn2 = fafb_seg_conn())
#'
# x ="annotation:ASB downseg 2"; tolerance = 0.25; name = "v14-seg neuron upload ASB";
# annotations = c("v14-seg upload", "ASB upseg 2"); avoid = "v14"; lock = TRUE;
# include.tags = TRUE; include.connectors = FALSE; downsample = 1;avoid.join = NULL;
# search.range.nm = 1000; duplication.range.nm=100; join = TRUE; join.tag = "TODO";
# fafbseg = TRUE; min_nodes = 2; return.uploaded.skids = TRUE;join.only=NULL
# pid = 1; conn = NULL; pid2 = 1; conn2 = fafb_seg_conn();include.potential.duplicates=FALSE
#'

#'
#' ### ASP-g project
#' annoatation.asp = "aSP-g L upstream"
# uploaded = catmaid_controlled_upload(x = annoatation.asp, tolerance = 0.5, name = "pheromonal circuit v14-seg upload ASB",
#                                      annotations = c("v14-seg upload", "ASB upseg", "ForBilly"), avoid = c("v14","duplicated"), lock = TRUE,
#                                      include.tags = TRUE, include.connectors = FALSE,
#                                      search.range.nm = 1000, join = TRUE, join.tag = "TODO",
#                                      fafbseg = TRUE, min_nodes = 2, downsample = 2,
#                                      brain = NULL, return.uploaded.skids = TRUE,
#                                      pid = 1, conn = NULL, pid2 = 1, conn2 = fafb_seg_conn())
#'
#'
#' ### Other
#' lns = read.neurons.catmaid("annotation:LH LN")
#' uploaded.lns = catmaid_upload_neurons(swc = pns, name = names(pns), annotations = "v14 LHN upload",
#'                                       include.tags = TRUE,include.connectors = TRUE,
#'                                       search.range.nm = 100, return.new.skids = TRUE,
#'                                       pid = 4, conn = local_conn(), max.upload = 10000)
#' uploaded.pns = catmaid_upload_neurons(swc = pns, name = names(pns), annotations = "v14 AL PN upload",
#'                                       include.tags = TRUE,include.connectors = TRUE,
#'                                       search.range.nm = 100, return.new.skids = TRUE,
#'                                       pid = 4, conn = local_conn(), max.upload = 10000)
#' uploaded = catmaid_controlled_upload(x ="annotation:ASB downseg", tolerance = 0.5, name = "v14-seg neuron upload",
#'                                      annotations = "v14-seg upload", avoid = "v14",
#'                                      include.tags = TRUE, include.connectors = TRUE,
#'                                      search.range.nm = 1000, join = TRUE, join.tag = "TODO",
#'                                      fafbseg = TRUE, min_nodes = 2,
#'                                      brain = NULL, return.uploaded.skids = TRUE,
#'                                      pid = 4, conn = local_conn(), pid2 = 1, conn2 = fafb_seg_conn())
#' uploaded = catmaid_upload_neurons(swc = frags, name ="neuron SWC upload", annotations = NULL,
#'                                   include.tags = TRUE,include.connectors = TRUE,
#'                                   search.range.nm = 100, return.new.skids = TRUE,
#'                                   pid = 4, conn = local_conn(), max.upload = 1000)
#' dels1 = catmaid_skids(x = "annotation:SWC upload",conn=local_conn(),pid=4)
#' dels2 = catmaid_skids(x = "annotation:neuron upload",conn=local_conn(),pid=4)
#' dels = c(dels1,dels2)
#' catmaid_delete_neurons(skid=dels,conn=local_conn(),pid=4, max.nodes = 500000, control = FALSE)
#'
#'
#' uploaded = catmaid_controlled_upload(x = "name:ASB Tester", join = TRUE, name = "ASB Tester from v14-seg",
#'                                      search.range.nm = 1000, annotations = "ASB Test v14-seg Upload", brain = elmr::FAFB14, lock = TRUE)
#' #' # Note that CATMAID links will also be supplied, so you can inspect a merge site in CATMAID. If you join in CATMAID, do not join in the interactive window, ad this will throw an errror, just keep hitting 'n' for no, until all options are exhausted.
#' #' # let's lock the neurons we just uploaded, to lessen the chance someone else will connect stuff to them and want to upload them AGAIN later...
#' catmaid_lock_neurons(skids = uploaded$downloaded.skids, conn = fafb_seg_conn())
#' #' # Oops, did you make a mistake in uploading this neuron?
#' delete.skids = catmaid::catmaid_skids("annotation:ASB Test v14-seg Upload")
#' catmaid_delete_neurons(delete.skids)
#' # Be careful wen deleting, especially if you have merged your fragment during the interactive join.
#' # Phew.
#'
#'
#'
#'
#'
#'
#'
#'
#'
#'
#'
#'
#' ### ASP-g v14-seg project
#' library(catnat)
#' # The first thing we can do it very quickly upload fragments that have no duplication, and make joins
# uploaded1 = catmaid_uncontrolled_upload(x ="annotation:aSP-g L upstream", tolerance = 0.05, name = "pheromonal circuit v14-seg upload ASB",
#                                         annotations = c("v14-seg upload", "ASB upseg", "ForBilly"), avoid = "v14", lock = TRUE,
#                                         include.tags = TRUE, include.connectors = FALSE, downsample = 1,
#                                         search.range.nm = 1000, duplication.range.nm=100, join = TRUE, join.tag = "TODO",
#                                         fafbseg = TRUE, min_nodes = 2, return.uploaded.skids = TRUE,
#                                         pid = 1, conn = NULL, pid2 = 1, conn2 = fafb_seg_conn())
#' uploaded2 = catmaid_controlled_upload(x = "aSP-g L upstream", tolerance = 0.05, name = "pheromonal circuit v14-seg upload ASB",
#'                                       annotations = c("v14-seg upload", "ASB upseg", "ForBilly"), avoid = "v14", lock = TRUE,
#'                                       include.tags = TRUE, include.connectors = FALSE, duplication.range.nm = 20,
#'                                       search.range.nm = 1000, join = TRUE, join.tag = "TODO",
#'                                       fafbseg = TRUE, min_nodes = 2, downsample = 2,
#'                                       brain = NULL, return.uploaded.skids = TRUE,
#'                                       pid = 1, conn = NULL, pid2 = 1, conn2 = fafb_seg_conn())
#' # If the duplication checking is too slow, you can only check a fraction of the nodes to speed things up.
#' # By setting downsample = 2, you will only check total nodes / 2, which still gives a good indication if a neuron is duplicated.
#' # Also setting fafbseg = FALSE is quicker, but means more false positive in terms if assessing potential duplication.
#'
#' catmaid_delete_neurons("annotation:ForBilly")
#' # If you accidentally uploas something that you did not mean to uploaded, or need to cut out a small bit of duplicated
#' # skeleton within CATMAID, tag it with something like "DELETE" and then use this function to delete such skeletons.
#' # You can only delete neurons fully comprised of nodes you can control, typically these are simply nodes you have
#' # placed or uploaded.
#'
#'
#' ## PNs
#' opns.em = read.neurons.catmaid("annotation:WTPN2017_AL_PN")
#' csd = as.character(catmaid_skids("annotation:^CSD$"))
#' oa = as.character(catmaid_skids("annotation:WTPN2017_OA_mALT_PN"))
#' csds.em = read.neurons.catmaid(csd)
#' oa.em = read.neurons.catmaid(oa)
#' opns.em = c(opns.em[!names(opns.em)%in%c(csd,oa)],csds.em,oa.em)
#'
alexanderbates/catnat documentation built on Sept. 5, 2023, 4:51 a.m.
rdrr.io home R language documentation Run R code online
CRAN packages Bioconductor packages R-Forge packages GitHub packages
Note that we can't provide technical support on individual packages. You should contact the package authors for that.
alexanderbates/catnat
Analyse neuronal morphology and connectivity (extends catmaid and nat)

R/editcatmaid.R
In alexanderbates/catnat: Analyse neuronal morphology and connectivity (extends catmaid and nat)

R Package Documentation

Browse R Packages

We want your feedback!

alexanderbates/catnat Analyse neuronal morphology and connectivity (extends catmaid and nat)

R/editcatmaid.R In alexanderbates/catnat: Analyse neuronal morphology and connectivity (extends catmaid and nat)

R Package Documentation

Browse R Packages

We want your feedback!

alexanderbates/catnat
Analyse neuronal morphology and connectivity (extends catmaid and nat)

R/editcatmaid.R
In alexanderbates/catnat: Analyse neuronal morphology and connectivity (extends catmaid and nat)
