#### Compile FCS files ----
#' Compile all .fcs files in a directory to a flowset
#'
#' A simple function to compile a directory of FCS files into a flowSet object using flowCore::read.flowSet().
#' Use the pattern argument to select only a subset of FCS files.
#'
#' @param data_dir Directory containing the .fcs files
#' @param pattern The pattern to use to find the files in the folder
#' @inheritParams flowCore::read.FCS
#' @family dataprep
#' @examples
#' \dontrun{
#' fcs <- compile_fcs(data_dir = "_data/raw", pattern = "\\.fcs")
#' }
#' @export
<- (data_dir,
= "\\.fcs",
column.pattern = ,
invert.pattern = ){
# Error checking
(data_dir %>% ("/")) data_dir <- data_dir %>% stringr::str_sub( = -2)
# Specifying files to use
<- (data_dir,
= ,
recursive = ,
full.names = ) %>%
()
(() == 0) ("No files found in folder \"", data_dir, "\"")
# Read the data files
(("Reading", (), "files to a flowSet.."))
fcs_raw <- %>%
flowCore::read.flowSet(transformation = ,
truncate_max_range = ,
emptyValue = ,
column.pattern = column.pattern,
invert.pattern = invert.pattern)
(fcs_raw)
}
#' Convert a flowSet into a tibble
#'
#' Use this function to convert a flowSet into the tibble object that the remaining
#' functions in cyCombine relies on.
#' A tibble is a Tidyverse implementation of a data.frame and can be treated as a such.
#' The majority of arguments revolves adding relevant info from the metadata file/object.
#' The panel argument is included to adjust the output column names using a panel with channel and antigen columns.
#' Bear in mind the column names will be altered with the following:
#' \code{stringr::str_remove_all("^\\d+\[A-Za-z\]+_") %>% stringr::str_remove_all("\[ _-\]")}.
#'
#'
#'
#' @param flowset The flowset to convert
#' @param metadata Optional: Can be either a filename or data.frame of the metadata file. Please give the full path from working directory to metadata file
#' @param sample_ids Optional: If a character, it should be the sample column in the metadata. If its a vector, it should have the same length as the total flowset. If NULL, sample ids will be the file names. If a single value, all rows will be assigned this value.
#' @param batch_ids Optional: If a character, it should be the column in the metadata containing the batch ids. If its a vector, it should have the same length as the total flowset. If a single value, all rows will be assigned this value.
#' @param filename_col Optional: The column in the metadata containing the fcs filenames. Needed if metadata is given, but sample_ids is not
#' @param condition Optional: The column in the metadata containing the condition. Will be used as the covariate in ComBat, but can be specified later. You may use this to add a different column of choice, in case you want to use a custom column in the ComBat model matrix.
#' @param anchor Experimental: The column in the metadata referencing the anchor samples (control references). Will be used as a covariate in ComBat, if specified. Please be aware that this column may be confounded with the condition column. You may use this to add a different column of choice, in case you want to use a custom column in the ComBat model matrix. You may use a custom column name, but it is good practice to add the name to the 'non_markers' object exported by cyCombine, to reduce the risk of unexpected errors.
#' @param down_sample If TRUE, the output will be down-sampled to size sample_size
#' @param sample_size The size to down-sample to
#' @param sampling_type The type of down-sampling to use. "random" to randomly select cells across the entire dataset, "batch_ids" to sample evenly (sample_size) from each batch, or "sample_ids" sample evenly (sample_size) from each sample.
#' @param seed The seed to use for down-sampling
#' @param clean_colnames (Default: TRUE). A logical defining whether column names should be cleaned or not. Cleaning involves removing isotope tags, spaces, dashes, and underscores.
#' @param panel Optional: Panel as a filename or data.frame. Is used to define colnames from the panel_antigen column
#' @param panel_channel Optional: Only used if panel is given. It is the column name in the panel data.frame that contains the channel names
#' @param panel_antigen Optional: Only used if panel is given. It is the column name in the panel data.frame that contains the antigen names
#' @family dataprep
#' @examples
#' \dontrun{
#' df <- convert_flowset(flowset = flowset,
#' metadata = file.path(data_dir, "metadata.csv"),
#' filename_col = "FCS_files",
#' sample_ids = "sample_id",
#' batch_ids = "batch_ids",
#' down_sample = FALSE)
#' }
#' @export
<- (flowset,
metadata = ,
filename_col = "filename",
sample_ids = ,
batch_ids = ,
= ,
anchor = ,
down_sample = ,
sample_size = 500000,
sampling_type = "random",
seed = 473,
clean_colnames = ,
panel = ,
panel_channel = "fcs_colname",
panel_antigen = "antigen"){
# Extract information necessary both with and without included metadata
## FlowSet row numbers
nrows <- flowCore::fsApply(flowset, )
## File names from flowset
<- flowset@phenoData %>%
() %>%
()
# Add metadata information (long)
(!(metadata)){
# Error handling
((filename_col)){
("Please specify a filename_col.")
}
("character" %in% (metadata)){
(!((metadata))){
("File \"", (metadata), "\" was not found")
}
# Get metadata
((metadata, suffix = ".xlsx")){
metadata <- ((metadata) %>%
readxl::read_xlsx())
} ((metadata, suffix = ".csv")){
metadata <- ((metadata) %>%
readr::read_csv())
} {
(stringr::str_c("Sorry, file", metadata, "is not in a supported format. Please use a .xlsx or .csv file.\n",
"Alternatively, a data.frame of the metadata can be used.",
sep = " "))
}
}
# Check for errors in metadata columns
md_cols <- (metadata)
cyCombine:::check_colname(md_cols, filename_col)
# Extract info from metadata
(!((metadata[filename_col]][1]), ".fcs")){
metadata[filename_col]] <- (metadata[filename_col]], ".fcs")
}
# Check that all metadata rows has a file
((metadata[filename_col]] %!in% )){
missing_files <- metadata[filename_col]][metadata[filename_col]] %!in% ]
(stringr::str_c("The sample ", missing_files, " in the metadata was not found in the provided folder and will be ignored.\n"))
# Remove files from metadata
metadata <- metadata[metadata[filename_col]] %in% ,]
}
# Check that all files are represented in metadata
(( %!in% metadata[filename_col]])){
missing_files <- [files %!in% metadata[filename_col]]]
(stringr::str_c("The sample ", missing_files, " was not found in the metadata file and will be ignored.\n"))
<- [files %in% metadata[filename_col]]]
nrows <- nrows[rownames(nrows) %in% ,] %>% ()
flowset <- flowset[files]
}
# Get sample ids
if ((sample_ids)){
sample_ids <- %>%
stringr::str_remove(".fcs") %>%
stringr::str_remove(".FCS") %>%
(nrows)
} if ((sample_ids) == 1){
cyCombine:::check_colname(md_cols, sample_ids)
sample_ids <- metadata[sample_ids]][match(, metadata[filename_col]])] %>%
stringr::str_remove(".fcs") %>%
stringr::str_remove(".FCS") %>%
(nrows)
}
# Get batch ids
if (!(batch_ids)){
((batch_ids) == 1){
cyCombine:::check_colname(md_cols, batch_ids)
batch_ids <- metadata[batch_ids]][match(, metadata[filename_col]])] %>%
() %>%
(nrows)
}
}
# Get condition
(!()){
(( == 1)){
cyCombine:::check_colname(md_cols, )
<- metadata[condition]][match(, metadata[filename_col]])] %>%
() %>%
(nrows)
}
}
# Get anchor
(!(anchor)){
((anchor == 1)){
cyCombine:::check_colname(md_cols, anchor)
anchor <- metadata[anchor]][match(, metadata[filename_col]])] %>%
() %>%
(nrows)
}
}
} { # If no metadata given
((sample_ids)){# Get sample ids from filenames
sample_ids <- %>%
stringr::str_remove(".fcs") %>%
stringr::str_remove(".FCS") %>%
(nrows)
}
}
## To down sample within fsApply
tot_nrows <- (nrows)
ids <- 1:tot_nrows
# Down sampling setup
(down_sample){
(seed)
(sampling_type == "random"){
(("Down sampling to", sample_size, "cells"))
<- (ids, sample_size) %>%
# Sorting here enables major resource savings when down-sampling
()
} (sampling_type == "batch_ids" & !(batch_ids)){ # even down-sampling from batches
(("Down sampling to", sample_size, "cells per batch"))
<- tibble::tibble(batch_ids, ids) %>%
dplyr::group_by(batch_ids) %>%
dplyr::slice_sample(n = sample_size) %>%
dplyr::pull(ids) %>%
()
} { # Even down-sampling from samples
(("Down sampling to", sample_size, "cells per sample"))
<- tibble::tibble(sample_ids, ids) %>%
dplyr::group_by(sample_ids) %>%
dplyr::slice_sample(n = sample_size) %>%
dplyr::pull(ids) %>%
()
}
# Down-sample metadata columns
ids <- ids[sample]
(!(sample_ids) & (sample_ids) > 1) sample_ids <- sample_ids[sample]
(!(batch_ids) & (batch_ids) > 1) batch_ids <- batch_ids[sample]
(!() & () > 1) <- [sample]
(!(anchor) & (anchor) > 1) anchor <- anchor[sample]
}
("Extracting expression data..")
fcs_data <- flowset %>%
purrr::when(down_sample ~ flowCore::fsApply(., cyCombine:::fcs_sample,
= ,
nrows = nrows),
~ flowCore::fsApply(., Biobase::exprs)) %>%
tibble::as_tibble() %>%
dplyr::mutate(id = ids) %>%
dplyr::select(id, dplyr::everything())
# Clean column names
if (!(panel)){
("character" %in% (panel)){
((panel, suffix = ".xlsx")){
panel <- ((panel) %>%
readxl::read_xlsx())
} ((panel, suffix = ".csv")){
panel <- ((panel) %>%
readr::read_csv())
} {
(("Sorry, file", panel, "is not in a supported format. Please use a .xlsx or .csv file."))
}
}
cols <- ((fcs_data), panel[panel_channel]]) %>%
.[!(.)]
col_names <- panel[panel_antigen]][cols]
fcs_data <- fcs_data %>%
dplyr::select(id, dplyr::all_of(panel[panel_channel]][cols]))
}{
col_names <- flowset[1]] %>%
flowCore::parameters() %>%
Biobase::pData() %>%
dplyr::pull(desc)
}
(clean_colnames) {
col_names <- col_names %>%
stringr::str_remove_all("^\\d+[A-Za-z]+_") %>%
stringr::str_remove_all("[ _-]")
}
(fcs_data) <- ("id", col_names)
# Add optional columns
(!(sample_ids)) fcs_data$ <- sample_ids
(!(batch_ids)) fcs_data$batch <- batch_ids
(!()) fcs_data$ <-
(!(anchor)) fcs_data$anchor <- anchor
("Your flowset is now converted into a dataframe.")
(fcs_data)
}
#' Extract from a flowset given a sample of indices
#' @noRd
fcs_sample <- (flowframe, , nrows, seed = 473){
# Determine which flowframe was given
ff_name <- flowCore::keyword(flowframe)$FILENAME %>%
()
ff_number <- ((nrows) == ff_name)
# Down-sample based on accumulated nrows (ensures the correct rows are extracted from each flowframe)
nrows_acc <- (0, nrows %>%
purrr::accumulate(`+`))
ff_sample <- - nrows_acc[ff_number]
ff_sample <- ff_sample[ff_sample > 0]
ff_sample <- ff_sample[ff_sample <= nrows[ff_number]]
# Extract expression data
ff <- flowframe %>%
Biobase::exprs()
ff <- ff[ff_sample, ]
(ff)
}
#### Data transformation ----
#' Transform data using asinh
#'
#' Inverse sine transformation of a tibble.
#' This function can also de-randomize data.
#'
#' @param df The tibble to transform
#' @param markers The markers to transform on
#' @param cofactor The cofactor to use when transforming
#' @param derand Derandomize. Should be TRUE for CyTOF data, otherwise FALSE.
#' @param .keep Keep all channels. If FALSE, channels that are not transformed are removed
#' @family dataprep
#' @examples
#' \dontrun{
#' uncorrected <- df %>%
#' transform_asinh(markers = markers)
#' }
#' @export
<- (,
markers = ,
cofactor = 5,
derand = ,
.keep = ){
((markers)){
markers <- %>%
cyCombine::()
}
((markers %!in% ())){
mes <- stringr::str_c("Not all given markers are in the data.\nCheck if the markers contain a _ or -:",
stringr::str_c(markers, collapse = ", "),
"Columns:",
stringr::str_c((), collapse = ", "),
sep = "\n"
)
(mes)
} (.keep & (() %!in% (()))){
("Your data contains non-unique column names. Please ensure they are unique. The column names are: ", stringr::str_c((), collapse = ", "))
}
(("Transforming data using asinh with a cofactor of ", cofactor, ".."))
transformed <- %>%
purrr::when(.keep ~ .,
~ dplyr::select_if(., (.) %in% (markers, ))) %>%
# Transform all data on those markers
dplyr::mutate(dplyr::across(dplyr::all_of(markers),
.fns = (x){
(derand) ((x)/cofactor) (x/cofactor)
}))
(transformed)
}
#### Wrapper function ----
#' Prepare a directory of .fcs files
#'
#' This is a wrapper function that takes you from a directory of .fcs files or a flowset to a transformed tibble.
#'
#'
#' @param flowset Optional: Prepare a flowset instead of a directory of fcs files
#' @inheritParams compile_fcs
#' @inheritParams convert_flowset
#' @inheritParams transform_asinh
#' @param transform If TRUE, the data will be transformed; if FALSE, it will not.
#' @family dataprep
#' @examples
#' \dontrun{
#' uncorrected <- data_dir %>%
#' prepare_data(metadata = "metadata.csv",
#' markers = markers,
#' filename_col = "FCS_name",
#' batch_ids = "Batch",
#' condition = "condition",
#' down_sample = FALSE)
#' }
#' @export
<- (data_dir = ,
flowset = ,
markers = ,
= "\\.fcs",
metadata = ,
filename_col = "filename",
sample_ids = ,
batch_ids = ,
= ,
anchor = ,
down_sample = ,
sample_size = 500000,
sampling_type = "random",
seed = 473,
panel = ,
panel_channel = "fcs_colname",
panel_antigen = "antigen",
= ,
cofactor = 5,
derand = ,
.keep = ,
clean_colnames = ){
# Stop if no data is given
((data_dir) & (flowset)) ("No data given.")
(!(data_dir)){
# Remove slash at end of data_dir
(data_dir %>% ("/")) data_dir <- data_dir %>% stringr::str_sub( = -2)
# Compile directory to flowset
((flowset)){
flowset <- data_dir %>%
cyCombine::( = )
}
# Look for metadata in data_dir
(!(metadata)){
(!"data.frame" %in% (metadata)){
(!((metadata)) & ((data_dir, metadata))) metadata <- (data_dir, metadata)
}
}
}
# Convert flowset to dataframe
fcs_data <- flowset %>%
cyCombine::(metadata = metadata,
filename_col = filename_col,
sample_ids = sample_ids,
batch_ids = batch_ids,
= ,
anchor = anchor,
down_sample = down_sample,
sample_size = sample_size,
sampling_type = sampling_type,
seed = seed,
panel = panel,
panel_channel = panel_channel,
panel_antigen = panel_antigen,
clean_colnames = clean_colnames) %>%
# Transform dataset with asinh
purrr::when( ~ cyCombine::(., markers = markers,
cofactor = cofactor,
derand = derand,
.keep = .keep),
~ .)
("Done!")
(fcs_data)
}
