TabulationAutomation: Auomated analysis and tabulation of results

Documented in factorAnalyzeR

#' Factor Analysis Functions
#'
#' @param modelFactorsList List of vectors of factors for each model
#'                         List elements: Vectors corresponding to models;
#'                         Vector elements: Each is a string with one factor name for that model
#'                 (use addModelContent function; see documentation using ?TabulationAutomation::addModelContent)
#' @param factorIndicatorsList List of vectors of factor indicators
#'                             List elements = vectors corresponding to models;
#'                             Vector elements = each factor's indicators smushed into one string
#'                                   ***(indicators separated by spaces or dashes for consecutive naming, e.g. "a1-a4 b1 b4")
#' @param varSpec Specify list of lists of factor and indicator variance specifications
#'                         Should have names of list = models; names of sublists = factors with variance specifications (e.g. f1@@1);
#'                         elements of sublists = vectors of indicators with specifications (e.g. c1*)
#' @param factorVarSpec Specify "free" if factors "free" to have different variances or "standardized" to fix at one
#'                   (can also specify a string to fix each factor according to custom MPlus syntax (e.g. "@@0"))
#' @param indVarSpec Variances for factor indicators. Specify "free" for syntax like f BY a*. Or can specify custom string. (e.g. @@1)
#' @param categoricalInds Vectors (elements = models):
#'                      NULL if all indicators continuous for that model
#'                      If model has ANY categorical indicators, string of categorical indicators' names in dataset
#'                          ***Indicators separated by spaces or dashes for consecutive naming,
#'                          e.g., for 2 models; one all continuous, one with categorical a1, a2, a3, b1, b4: c(NULL, "a1-a3 b1 b4")
#' @param na.string What value indicates missing variables in the data MPlus will analyze? (Default "-9999")
#' @param mplusDataFilePath Full file path for the data MPlus will analyze?
#' @param mplusOutputDirectory Where should the MPlus syntax template and Mplus .inp and .out files go?
#' @param templateName What should the template created by this function be called? (specify without ".txt" file extension)
#' @param run TRUE if you want to run the syntax created by the template
#' @param starts String with number of initial stage starts and final stage optimizations, e.g., "100 20"
#' @param iterations Initial stage iterations, e.g. "10"
#' @param cores Number of logical processors to use (MandelBot = 64)
#' @param monitor "on" or "off" for MPlus monitor option
#' @param outputTechOptions e.g. 'TECH1 TECH3 TECH4 TECH8 TECH10 residual standardized cinterval patterns'
#' @param integration e.g. 'INTEGRATION = montecarlo'
#' @param extraUseVars For instance, if you wanted to include more than just factor indicators, like subpopulation identifiers
#' @param variableExtras e.g. 'cluster is PSU;
#'                           stratification is STRATUM;
#'                                 weight is WEIGHT;'
#' @param analysisExtras Any extra syntax for the ANALYSIS command
#' @param type e.g. "complex;", "mixture;", etc.
#' @return
#' @export
#'
#' # Can make into examples at some point
#'
#' TabulationAutomation::addModelContent(setName = 'dsm5', list = 'factorContentsList', # Factor Model 1
#'            setContent = c("b1-b5", "c1-c2", "d1-d7 e1-e2 e5-e6", "e3-e4"),
#'                        # using indicator variables as vector elements
#'            setNameAttribute = c('INTRUSIONS', 'AVOIDANCE', 'COGMOOD', 'HYPERAROUSAL'))
#'                        # using factor variables as names
#'
#' TabulationAutomation::addModelContent(setName = 'dysphoria', list = 'factorContentsList',
#'            setContent = c("b1-b5", "c1-c2", "d1-d7", "e1-e2 e5-e6", "e3-e4"),
#'            setNameAttribute = c('INTRUSIONS', 'AVOIDANCE', 'DYSPHORIA', 'HYPERAROUSAL'))
#'
#' TabulationAutomation::addModelContent(setName = 'dsm5', list = 'modelContentsList',
#'            setContent = c('INTRUSIONS', 'AVOIDANCE', 'COGMOOD', 'HYPERAROUSAL'),
#'                       # here, using real factor variables as vector elements
#'            setNameAttribute = c('Intrusions', 'Avoidance', 'Cognition/Mood', 'Hyperarousal'))
#'                       # here, using "pretty" labels as names
#'
#' TabulationAutomation::addModelContent(setName = 'dysphoria', list = 'modelContentsList',
#'            setContent = c('INTRUSIONS', 'AVOIDANCE', 'DYSPHORIA', 'HYPERAROUSAL'),
#'            setNameAttribute = c('Intrusions', 'Avoidance', 'Dysphoria', 'Hyperarousal'))
#'
#' makeMPlus(modelFactorsList = modelContentsList, # if factor scores are predictors in regression analyses,
#'               # ...modelFactorsList will be the same as what you're using
#'                         # (1) for "modelPredictorsList" in the autoAnalyze function
#'                         # and (2) maybe for "list" argument in addModelContent
#'                                  # if you used that to make whatever object (here, "modelContentsList")
#'                                  # passed to "modelPredictorsList"
#'               # so see above example
#'           factorIndicatorsList = factorContentsList,
#'           factorVarSpec = 'free', categoricalInds = FALSE, na.string = '-9999',
#'           mplusDataFilePath = paste0(getwd(), '//subdirectory//data.csv'), mplusOutputDirectory = getwd(),
#'           templateName = 'cfa_template', run = FALSE)
factorAnalyzeR <- function(modelFactorsList, factorIndicatorsList,
                           varSpec = NULL, factorVarSpec = 'free', indVarSpec = '@1',
                           categoricalInds = NULL, na.string = '-9999',
                           mplusDataFilePath, mplusDataFileColumnNames,
                           templateName = 'MPLUStemplate', mplusOutputDirectory = getwd(),
                           run = FALSE, starts = '50 20', iterations = '10', cores = 8,
                           monitor = 'off', outputTechOptions = 'TECH1 TECH3 TECH8',
                           integration = 'montecarlo', estimator = 'mlr',
                           extraUseVars = NULL,
                           variableExtras = NULL, analysisExtras = NULL,
                           type = NULL) {

  require(MplusAutomation); require(plyr); require(dplyr); require(MplusAutomationFork);

  try(source("/home/sean/R/x86_64-pc-linux-gnu-library/3.4/MplusAutomationFork"))

  templateFilePath <- paste0(mplusOutputDirectory, '/', templateName, '.txt')

  factorNames_model <- function(tFile = templateFilePath) {
    force(tFile)
    eval(parse(text = paste0("cat('\"', modelFactorsList[[", paste0(1:length(modelFactorsList),
                             collapse = "]], '\" \"', modelFactorsList[["), "]], '\"',
                             file = tFile, append = TRUE)")))
  }

### Detect NULL arguments and also see how MPlus-specific user has been in specifying arguments.
  # Add syntactic necessities if missing

  if (is.null(integration)) {
    integration <- ''  # blank if null
  } else {
    if (grepl('INTEGRATION', integration) == FALSE) {
      if (grepl('integration', integration) == FALSE) {
        message('Changing syntax for integration to include INTEGRATION = ...')
        integration <- paste0('INTEGRATION = ')  # add integration = if missing
      }
    }
    if (grepl(';', integration) == FALSE) integration <- paste0(integration, ' ;') # add ; to end line if missing
  }

  if (is.null(starts)) {
    starts <- ''  # blank if null
  } else {
    if (grepl('STARTS', starts) == FALSE) {
      if (grepl('starts', starts) == FALSE) {
        message('Changing syntax for starts to include STARTS = ...')
        starts <- paste0('STARTS = ')  # add STARTS = if missing
      }
    }
    if (grepl(';', starts) == FALSE) starts <- paste0(starts, ' ;') # add ; to end line if missing
  }

  if (is.null(iterations)) {
    iterations <- ''  # blank if null
  } else {
    if (grepl('STIT', iterations) == FALSE) {
      if (grepl('STIT', iterations) == FALSE) {
        message('Changing syntax for iterations to include STIT =  ...')
        iterations <- paste0('STIT = ')  # add STIT =  if missing
      }
    }
    if (grepl(';', STIT) == FALSE) iterations <- paste0(STIT, ' ;') # add ; to end line if missing
  }

  if (!is.null(variableExtras)) if (grepl(';', variableExtras) == FALSE) variableExtras <- paste0(variableExtras, '; ')
  if (!is.null(analysisExtras)) if (grepl(';',  analysisExtras) == FALSE)  analysisExtras <- paste0(analysisExtras, '; ')
  if (!is.null(extraUseVars)) if (grepl(';',  extraUseVars) == FALSE)  extraUseVars <- paste0(extraUseVars, '; ')
  if (!is.null(type)) {
    if (grepl(';',  type) == FALSE)  type <- paste0(type, '; ')
    if (grepl('TYPE', type) == FALSE) {
      if (grepl('type', type) == FALSE) {
        message('Changing syntax for type to include TYPE =  ...')
        type <- paste0('TYPE = ', type)  # add TYPE =  if missing
      }
    }
  } else { type <- '' } # blank if null

#=============================================================================================================#
##### Function that Appends Syntax for varNames#model Iterator ####
# Defines which variables are used in each factor model and which (if any) are categorical
#=============================================================================================================#

  varDefs_model <- function(outDir, tFileName, fList, indicatorsList, categorical) {

    force(tFileName); force(outDir); force(fList); force(indicatorsList); force(categorical)
  # If specified single string "all", means all indicators for all models categorical...
    # so re-make categorical (i.e. "categoricalInds" from wrapper function)...
    # ...into a vector with "all" for each entry

### For each model...
    for (m in 1:length(modelContentsList)) {
      indicators <- indicatorsList[[m]] # vector of indicator-set strings for each factor
      indicatorString <- paste0(indicators, collapse = ' ')
  # USEVARIABLES
      # Names of dataset variables used = string of indicators in that model
      #varLines <- paste0("USEVARIABLES ARE ", indicatorString, collapse = ' ') # paste indicators into one string
  # CATEGORICAL ARE
      if (!is.null(categorical)) {  # if model has any categorical indicators
        if (length(categorical) == 1) categorical <- rep(categorical, length(fList)) # if only 1 string, applies to all models
        if ('all' %in% indicators) {
          stop('Error: "all" is reserved for internal and MPlus functions and cannot be used as an indicator name.')
        }
        if (categorical[m] == 'all') { # if all indicators for that model are categorical
          catLines <- paste0(paste0('CATEGORICAL ARE ',
                                    indicatorString, collapse = ' '), # just paste all indicators for that model
                             ';', collapse = ' ')
        } else { # if a specific subset of indicators given by a string
          catLines <- paste0('CATEGORICAL ARE ', paste0(categorical[m], collapse = ' '), '; ')
        }

        cat(c("\"", catLines, "\" \n"),
            file = paste0(outDir, '/', tFileName, '.txt'), append = TRUE, sep = "")
      }
    }
  }

#=============================================================================================================#
##### Function that Appends Syntax for factorDefs#model Iterator ####
  # Defines Which Indicators Load onto Which Factors
#=============================================================================================================#

  ## By statements (for factor indicators) and variances
  factorDefs_model <- function(outDir, tFileName, fList, indicatorsList, varSpecCustom = NULL, fVariances, indVariances) {

    force(fList); force(indicatorsList); force(outDir); force(tFileName); force(fVariances); force(indVariances)

    for (m in 1:length(fList)) {


  #### "BY" Statements ####

      factors <- fList[[m]] # model-specific vector of factor names
      indicators <- indicatorsList[[m]] # model-specific string of indicators

    # If very specific/heterogeneous indicator variances by factor, user should have passed a list to the indVarSpec argument
      # That argument from the wrapper will be passed to the indVariances argument in this internal function
      # This code is analogous to that specifying the factor variances statements below
      if (is.list(indVariances)) {
        iVars <- indVariances[[m]]
        iVars <- gsub(x = iVars, pattern = 'free', replacement = '*')
      } else {
        if (length(indVariances) > 1) iVars <- indVariances[m]
        if (length(indVariances == 1)) iVars <- indVariances
      }

      if (length(iVars) == 1) {
        if (iVars == 'free') {
          iVarsSpec <- paste0(indicators, sep = "*; ")
        } else {
          if (iVars == 'standardized' | iVars == 'standard') {
            iVarsSpec <- paste0(indicators, sep = "@1; ")
          } else {
            iVarsSpec <- paste0(indicators, sep = paste0(iVars, "; "))
          }
        }
      } else {
        iVarsSpec <- paste0(indicatorsList[[names(indicatorsList)[m]]], paste0(iVars, '; '))
      }

      factorsByIndicators <- paste0(factors, ' BY ', iVarsSpec, '; ') # "BY" statements (i.e. (a) which indicators load on each factor and (b), indicator variances)

  ### Factor Variances Statements ####
    ## Specify if factor variances are free, fixed at 1, or fixed at something else
      # if fVars is just a string, assume that string applies to all factors for this model
        # (implies factorVarSpec argument in wrapper fx is vector, meaning models have different specs, but factors within model have same spec)

      if (!is.null(varSpecCustom)) { # If argument varSpec passed as list of list of factor and indicator variance specifications
        factorsByIndicators <- paste0(paste0(gsub(pattern = '@.*', replacement = '', names(unlist(varSpecCustom[[m]]))), ' BY ', unlist(varSpecCustom[[m]])), collapse = '; ')
        fVarsSpec <- paste0(paste0(unique(gsub(x = gsub('.*[.]', '', names(unlist(varSpecCustom[[m]], recursive = FALSE))), pattern = ';.*', replacement = ''), collapse = '; '), '; '))
                    # just the internal model and variance specification from varSpecCustom list
      } else {

        # If very specific/heterogeneous factor variances by model, user should have passed a list to the fVarSpec argument
        # That argument from the wrapper will be passed to the fVariances argument in this internal function
        if (is.list(fVariances)) {
          fVars <- fVariances[[m]]
          if (grepl(pattern = 'free', fVars)) fVars <- gsub(x = fVars, pattern = 'free', replacement = '*')
        } else {
          if (length(fVariances) > 1) fVars <- fVariances[m]
          if (length(fVariances == 1)) fVars <- fVariances
        }

        if (length(fVars) == 1) {
          if (fVars == 'free') {
            fVarsSpec <- paste0(fList[[names(fList)[m]]], sep = "*; ")
          } else {
            if (fVars == 'standardized' | fVars == 'standard') {
              fVarsSpec <- paste0(fList[[names(fList)[m]]], sep = "@1; ")
            } else {
              fVarsSpec <- paste(fList[[names(fList)[m]]], sep = paste0(fVars, "; "))
            }
          }
        } else {
          fVarsSpec <- paste0(fList[[names(fList)[m]]], paste0(fVars, '; '))
        }
      }

      cat(c("\"", factorsByIndicators, ' ; ', fVarsSpec, "\" "),
          file = paste0(outDir, '/', tFileName, '.txt'), append = TRUE, sep = "")

    }
  }

#### Make MPlus syntax ####

cat(file = paste0(mplusOutputDirectory, '/', templateName, '.txt'), '[[init]]\n\n')

cat(file = paste0(mplusOutputDirectory, '/', templateName, '.txt'), append = TRUE,
'iterators = model;
model = ', paste0('1:', length(modelContentsList)), ';

modelNames#model =
  ', names(modelContentsList), ';

factorNames#model = ')
for (m in 1:length(factorContentsList)) {
  cat(c("\n     \"", names(factorContentsList[[m]]), "\" "),
      file = paste0(mplusOutputDirectory, '/', templateName, '.txt'), append = TRUE)
}

cat(file = paste0(mplusOutputDirectory, '/', templateName, '.txt'), append = TRUE, ';

useVars#model = ')
# USEVARIABLES
# Names of dataset variables used = string of indicators in that model
for (m in 1:length(factorContentsList)) {
  indicators <- factorIndicatorsList[[m]]
  # vector of indicator-set strings for each factor
  indicatorString <- paste0(indicators, collapse = ' ')
  varLines <- paste0("USEVARIABLES ARE ", indicatorString, " ", extraUseVars, "")
  cat(c("\n   \"", varLines, "\" "), file = paste0(mplusOutputDirectory, '/', templateName, '.txt'), append = TRUE)
}

cat(file = paste0(mplusOutputDirectory, '/', templateName, '.txt'), append = TRUE, ';

catVars#model =
    ')
varDefs_model(outDir = mplusOutputDirectory, tFileName = templateName,
              fList = modelFactorsList, indicatorsList = factorIndicatorsList, categorical = categoricalInds)

cat(file = paste0(mplusOutputDirectory, '/', templateName, '.txt'), append = TRUE, ';

factorDefs#model = ', sep = "")
factorDefs_model(outDir = mplusOutputDirectory, tFileName = templateName,
                 fList = modelFactorsList, indicatorsList = factorIndicatorsList,
                 varSpecCustom = varSpec, fVariances = factorVarSpec, indVariances = indVarSpec)


cat(file = paste0(mplusOutputDirectory, '/', templateName, '.txt'), append = TRUE, ';

filename =  "[[modelNames#model]].inp";
outputDirectory = "', mplusOutputDirectory, '"; \n\n')

cat(file = paste0(mplusOutputDirectory, '/', templateName, '.txt'), append = TRUE, '[[/init]]

DATA: FILE IS "', mplusDataFilePath, '";

VARIABLE:
  NAMES ARE ', mplusDataFileColumnNames, ';
  [[useVars#model]]
  [[catVars#model]]
  MISSING ARE ALL (', na.string, ');',
  variableExtras, '

ANALYSIS:
  ', type, '
  ESTIMATOR = ', estimator, '; ', integration, '
  PROCESS = ', cores, ';
  ', starts, ' ', iterations, ' ', analysisExtras, '

MODEL:
  [[factorDefs#model]];

OUTPUT:
  standardized ', outputTechOptions, ';

PLOT:
  TYPE = plot1;
  TYPE = plot2;
  TYPE = plot3;
  MONITOR = ', monitor, ';

SAVEDATA:
  FILE IS "FScores_[[modelNames#model]].csv";
  SAVE IS fscores;
  FORMAT IS free;

', sep = '')

  ### Create models from template
  templateFilePath <- paste0(mplusOutputDirectory, '/', templateName, '.txt')

  #MplusAutomation::createModels(templatefile = templateFilePath)
  modelCreatoR(templatefile = templateFilePath)

  ## Run models if run is set to TRUE
  if (run == TRUE) {
    MplusAutomation::runModels(mplusOutputDirectory, recursive = FALSE, replaceOutfile = "modifiedDate",
                               showOutput = TRUE)
  } else {
    warning('Set run = TRUE (default is false) if you want MPlus scripts to run.')
  }
}