In Thaliehln/timci: Data management and visualisation for the TIMCI project

library(qrcode)
knitr::opts_chunk$set(echo = FALSE,
                      warning = FALSE,
                      message = FALSE)

fu_start <- params$fu_start
fu_end <- params$fu_end
fu_vstart <- params$fu_vstart
fu_vend <- params$fu_vend
physicalfu_start <- params$physicalfu_start
physicalfu_end <- params$physicalfu_end
rctls_pid <- params$rctls_pid
fu_fid <- params$fu_fid
output_dir <- params$output_dir

# Create `odk_external_files` folder if it does not exist
dir.create(file.path(output_dir, "odk_external_files"), showWarnings = FALSE)

is_tanzania <- Sys.getenv('TIMCI_COUNTRY') == 'Tanzania'
is_india <- Sys.getenv('TIMCI_COUNTRY') == 'India'
is_day7fu <- fu_fid == Sys.getenv('TIMCI_CRF_DAY7_FID')
is_day7fu_tz <- is_tanzania & is_day7fu
is_day28fu <- fu_fid == Sys.getenv('TIMCI_CRF_DAY28_FID')
is_day28fu_tz <- is_tanzania & is_day28fu
is_day28fu_up <- is_india & is_day28fu

if (!is.null(params$facility_data)) {
  facility_data <- params$facility_data
  res <- timci::extract_enrolled_participants(facility_data)
  day0_data <- res[[1]]
  pii <- res[[2]]
  raw_fu_data <- params$raw_fu_data
  raw_oth_fu_data <- params$raw_oth_fu_data
  raw_withdrawal_data <- params$raw_withdrawal_data
  rct_ls_form_list <- params$rct_ls_form_list
}

fu <- timci::generate_fu_log_csv(pii,
                                 raw_fu_data,
                                 fu_start,
                                 fu_end,
                                 fu_vstart,
                                 fu_vend)

if (!is.null(raw_oth_fu_data)) {
  if (nrow(raw_oth_fu_data) > 0) {
    # Remove duplicate child ID entries in raw_oth_fu_data based while keeping the most recent date
    raw_oth_fu_data <- raw_oth_fu_data %>% rename(child_id = 'a1-pid')
    raw_oth_fu_data <- aggregate(date ~ child_id, raw_oth_fu_data, max)
    raw_oth_fu_data <- raw_oth_fu_data %>% rename('a1-pid' = 'child_id')
    # Add date of the day 7 follow-up in the day 28 log, if available
    if (nrow(fu) > 0){
      fu <- merge(fu,
                  raw_oth_fu_data[, c('a1-pid', 'date')],
                  by.x = 'name',
                  by.y = 'a1-pid',
                  all.x = TRUE) %>%
          dplyr::rename(day7fu = date)
    }
  }
}

physicalfu <- timci::generate_physical_fu_log_csv(pii = pii,
                                                  fudf = raw_fu_data,
                                                  wmin_nophone = physicalfu_start,
                                                  wmin_phone = physicalfu_start,
                                                  wmax = physicalfu_end,
                                                  vwmin = fu_vstart,
                                                  vwmax = fu_vend)

if (!is.null(raw_oth_fu_data)) {
  if (nrow(raw_oth_fu_data) > 0) {

    # Remove duplicate child ID entries in raw_oth_fu_data based while keeping the most recent date
    raw_oth_fu_data <- raw_oth_fu_data %>%
      dplyr::rename(child_id = 'a1-pid',
                    day7_phonenb = 'a1-phonenb_manual') %>%
      dplyr::group_by(child_id) %>%
      dplyr::filter(date == max(date)) %>%
      dplyr::ungroup() %>%
      arrange(child_id, is.na(day7_phonenb)) %>% 
      distinct(child_id, .keep_all = TRUE)  %>% 
      dplyr::rename('a1-pid' = 'child_id')

    # Add data from the day 7 follow-up (date, new phone number) in the day 28 log, if available
    selected_columns <- c('a1-pid', 'date', 'day7_phonenb')
    if (nrow(fu) > 0){
      fu <- merge(fu,
                  raw_oth_fu_data[, selected_columns],
                  by.x = 'name',
                  by.y = 'a1-pid',
                  all.x = TRUE) %>%
        dplyr::rename(day7fu = date) %>%
        dplyr::arrange(fid, submitter, enroldate = as.Date(enroldate, "%Y-%m-%d"))
    }

    # Add data from the day 7 follow-up (date, new phone number) in the day 28 log, if available
    if (nrow(physicalfu) > 0){
      physicalfu <- merge(physicalfu,
                    raw_oth_fu_data[, selected_columns],
                    by.x = 'name',
                    by.y = 'a1-pid',
                    all.x = TRUE) %>%
        dplyr::rename(day7fu = date) %>%
        dplyr::arrange(fid, !is.na(day7fu), enroldate = as.Date(enroldate, "%Y-%m-%d"))
    }

  }
}

if (nrow(raw_fu_data) > 0) {
  failed_fu_data <- timci::format_day7_data(raw_fu_data)[[2]]
  if (fu_fid == Sys.getenv('TIMCI_CRF_DAY7_FID')) {
    tmp <- timci::export_df2xlsx(failed_fu_data, output_dir, "day7_unsuccessful_calls")
  } else if (params$fu_fid == Sys.getenv('TIMCI_CRF_DAY28_FID')) {
    tmp <- timci::export_df2xlsx(failed_fu_data, output_dir, "day28_unsuccessful_calls")
  }
}

ltfu <- NULL

ltfu <- timci::generate_ltfu_log(day0_data,
                                 raw_fu_data,
                                 fu_end)

tmp <- timci::export_df2xlsx(ltfu, output_dir, "day7_lost_to_followup")

tmp <- timci::export_df2xlsx(ltfu, output_dir, "day28_lost_to_followup")

pii_cols <- c("child_id",
              "fs_name",
              "ls_name",
              "cg_fs_name",
              "cg_ls_name",
              "mother_fs_name",
              "mother_ls_name",
              "phone_nb",
              "phone_owner",
              "phone_nb2",
              "cmty_contact")
ext_ltfu <- merge(ltfu, pii[, pii_cols], by = c("child_id"))
if (fu_fid == Sys.getenv('TIMCI_CRF_DAY7_FID')) {
  tmp <- timci::export_df2xlsx(ext_ltfu, output_dir, "day7_lost_to_followup_incl_pii")
  tmp <- timci::export_df2xlsx(ext_ltfu %>% dplyr::filter(is.na(fu_attempts)),
                               output_dir,
                               "day7_lost_to_followup_no_attempt_incl_pii")
} else if (params$fu_fid == Sys.getenv('TIMCI_CRF_DAY28_FID')) {
  tmp <- timci::export_df2xlsx(ext_ltfu, output_dir, "day28_lost_to_followup_incl_pii")
}

csv_filename <- file.path(output_dir, "odk_external_files", "day7fu.csv")
write.csv(fu, csv_filename, row.names = FALSE, quote = FALSE, fileEncoding = "UTF-8")

physicalfu_csv_filename <- file.path(output_dir, "odk_external_files", "day7physicalfu.csv")
write.csv(physicalfu, physicalfu_csv_filename, row.names = FALSE, quote = FALSE, fileEncoding = "UTF-8")

csv_filename <- file.path(output_dir, "odk_external_files", "day28fu.csv")
write.csv(fu, csv_filename, row.names = FALSE, quote = FALSE, fileEncoding = "UTF-8")

physicalfu_csv_filename <- file.path(output_dir, "odk_external_files", "day28physicalfu.csv")
write.csv(physicalfu, physicalfu_csv_filename, row.names = FALSE, quote = FALSE, fileEncoding = "UTF-8")

Upload list of participants on the ODK Central server

form_url <- "N/A"
draft_response <- NULL
csv_upload_response <- NULL
new_version <- "N/A"
publication_response <- NULL

current_version <- "N/A"
if (fu_fid %in% rct_ls_form_list) {
  # Get form details
  cform <- ruODK::form_detail(pid = rctls_pid,
                              fid = fu_fid,
                              url = ruODK::get_default_url(),
                              un = ruODK::get_default_un(),
                              pw = ruODK::get_default_pw())
  version_no_bracket <- strsplit(cform$version, "[[]")[[1]][1]
  encryption_version <- strsplit(cform$version, "[[]")[[1]][2]
  current_version <- as.numeric(version_no_bracket)
}

form_url <- paste0(ruODK::get_default_url(), "/v1/projects/", rctls_pid, "/forms/", fu_fid)
# Create a new draft of the form
draft_response  <- timci::create_odkc_draft_form(ruODK::get_default_un(),
                                                 ruODK::get_default_pw(),
                                                 form_url)
# Upload the new CSV created as a form attachment
csv_upload_response <- timci::upload_odkc_csv_attachment(ruODK::get_default_un(),
                                                         ruODK::get_default_pw(),
                                                         form_url,
                                                         csv_filename)

r form_url

Initial version of the form: r current_version
Create a new draft of the form: r draft_response$success
Upload the CSV containing the current follow-up log as an attachment to the draft: r csv_upload_response$success

# Upload the new CSV for physical follow-up created as a form attachment
physicalfu_csv_upload_response <- timci::upload_odkc_csv_attachment(ruODK::get_default_un(),
                                                                    ruODK::get_default_pw(),
                                                                    form_url,
                                                                    physicalfu_csv_filename)
cat(paste0("* Upload the CSV containing the current physical follow-up log as an attachment to the draft: **", physicalfu_csv_upload_response$success, "**"))

# Publish a draft form
new_version <- as.character(current_version + 0.000000001)
# Reintegrate the encryption number if necessary
if (!is.na(encryption_version)) {
  new_version <- paste0(new_version, "[", encryption_version)
}
publication_response <- timci::publish_odkc_draft_form(ruODK::get_default_un(),
                                                       ruODK::get_default_pw(),
                                                       form_url,
                                                       new_version)