In gabrielodom/DMRcomparePaper: Comparing Different Analytical Tools for Identifying Differentially Methylated Regions (DMRs) in Illumina Methylation Arrays

knitr::opts_chunk$set(echo = TRUE)

Load and Inspect the Results

We have saved the tabulated results files for each method in the DMRcompare package. We will use the tidyverse package suite for data management utility. Access these via

library(DMRcompare)
library(tidyverse)
data("dmrcateRes_df")
data("probeLassoRes_df")
data("bumphunterRes_df")
data("combpRes_df")

These data frames all contain the following common elements:

commonNames_char <- Reduce(
  intersect,
       list(colnames(dmrcateRes_df),
          colnames(probeLassoRes_df),
          colnames(bumphunterRes_df),
          colnames(combpRes_df))
)
commonNames_char

Common Columns

The common elements of interest are

• Performance metrics: false negatives (FN), false positives (FP), true negatives (TN), true positives (TP), power, the sample size used to calculate power (nPower), the area under the precision-recall curve (AuPR), precision false positives (FPprecis), precision true positives (TPprecis), precision, the sample size used to calculate precision (nPrecis), Matthews Correlation Coefficient (mcc), and the F1-score (F1). Because many of these performance metric are related to each other arithmetically, we will reduce our inspection set to AuPR, MCC, and F1.
• Parameters: the DMR-detection method used (method), the simulated effect size (delta), and the random seed to ensure reproducibility (seed).

Method-Specific Columns

The method-specific parameters are

• DMRcate: lambda and C

setdiff(colnames(dmrcateRes_df), commonNames_char)

• ProbeLasso: adjPval, mLassoRad, and minDmrSep

setdiff(colnames(probeLassoRes_df), commonNames_char)

• Bumphunter: cutoffQ and maxGap

setdiff(colnames(bumphunterRes_df), commonNames_char)

• Comb-p: combSeed and combDist

setdiff(colnames(combpRes_df), commonNames_char)

Table of All Parameter Settings

We finally have need for the supplemental tables of all performance metrics for all tested parameter settings under each delta while averaged over each seed.

DMRcate

This table shows the average performance of the DMRcate method at each $\delta > 0$.

dmrcate_tab <-
  dmrcateRes_df %>%
  select(delta, seed, lambda, C,
         TP, FP, FN, power, precision, AuPR, mcc, F1) %>% 
  filter(delta > 0) %>% 
  group_by(delta, lambda, C) %>%
  summarise(
    TP     = CalcMeanSD(TP, sigFigsMean = 0),
    FP     = CalcMeanSD(FP, sigFigsMean = 0),
    FN     = CalcMeanSD(FN, sigFigsMean = 0),
    Pwr    = CalcMeanSD(power),
    Precis = CalcMeanSD(precision),
    AuPR   = CalcMeanSD(AuPR),
    MCC    = CalcMeanSD(mcc),
    F1     = CalcMeanSD(F1)
  )

# write_csv(dmrcate_tab, path = "../resultsData/DMRcate_total_results.csv")
dmrcate_tab %>% 
  kable()

ProbeLasso

This table shows the average performance of the ProbeLasso method at each $\delta > 0$.

pl_tab <-
  probeLassoRes_df %>%
  select(delta, seed, adjPval, mLassoRad, minDmrSep,
         TP, FP, FN, power, precision, AuPR, mcc, F1) %>% 
  filter(delta > 0) %>% 
  group_by(delta, adjPval, mLassoRad, minDmrSep) %>%
  summarise(
    TP     = CalcMeanSD(TP, sigFigsMean = 0),
    FP     = CalcMeanSD(FP, sigFigsMean = 0),
    FN     = CalcMeanSD(FN, sigFigsMean = 0),
    Pwr    = CalcMeanSD(power),
    Precis = CalcMeanSD(precision),
    AuPR   = CalcMeanSD(AuPR),
    MCC    = CalcMeanSD(mcc),
    F1     = CalcMeanSD(F1)
  )

# write_csv(pl_tab, path = "../resultsData/ProbeLasso_total_results.csv")
pl_tab %>% 
  kable()

Bumphunter

This table shows the average performance of the Bumphunter method at each $\delta > 0$.

bump_tab <-
  bumphunterRes_df %>%
  select(delta, seed, cutoffQ, maxGap,
         TP, FP, FN, power, precision, AuPR, mcc, F1) %>% 
  filter(delta > 0) %>% 
  group_by(delta, cutoffQ, maxGap) %>%
  summarise(
    TP     = CalcMeanSD(TP, sigFigsMean = 0),
    FP     = CalcMeanSD(FP, sigFigsMean = 0),
    FN     = CalcMeanSD(FN, sigFigsMean = 0),
    Pwr    = CalcMeanSD(power),
    Precis = CalcMeanSD(precision),
    AuPR   = CalcMeanSD(AuPR),
    MCC    = CalcMeanSD(mcc),
    F1     = CalcMeanSD(F1)
  )

# write_csv(bump_tab, path = "../resultsData/Bumphunter_total_results.csv")
bump_tab %>% 
  kable()

Comb-p

This table shows the average performance of the Comb-p method at each $\delta > 0$.

comb_tab <-
  combpRes_df %>%
  select(delta, method, seed, combSeed, combDist,
         TP, FP, FN, power, precision, AuPR, mcc, F1, time) %>% 
  filter(delta > 0) %>% 
  group_by(delta, combSeed, combDist) %>%
  summarise(
    method = "Comb-p",
    TP     = CalcMeanSD(TP, sigFigsMean = 0),
    FP     = CalcMeanSD(FP, sigFigsMean = 0),
    FN     = CalcMeanSD(FN, sigFigsMean = 0),
    Pwr    = CalcMeanSD(power),
    Precis = CalcMeanSD(precision),
    AuPR   = CalcMeanSD(AuPR),
    MCC    = CalcMeanSD(mcc),
    F1     = CalcMeanSD(F1),
    time   = CalcMeanSD(time, sigFigsMean = 0)
  )

# write_csv(comb_tab, path = "../resultsData/Combp_total_results.csv")
comb_tab %>% 
  select(-method) %>% 
  kable()

Table of Best-Performing Parameters

DMRcate

This method shows best performance with lambda = 500 and C = 5.

resDMRc3_df <- 
  dmrcate_tab %>% 
  filter(lambda == 500) %>% 
  filter(C == 5) %>% 
  ungroup() %>% 
  select(-one_of("lambda", "C"))
DMRcTimes_df <-
  lilyCompTimes_df %>% 
  filter(Method == "DMRcate") %>% 
  mutate(time = paste0(round(Mean, 0), " (", round(StdDev, 2), ")")) %>% 
  select(-one_of("Mean", "StdDev")) %>% 
  rename("delta" = "Delta")

res3_ls$DMRcate <-
  resDMRc3_df %>%
  left_join(DMRcTimes_df, by = "delta") %>% 
  select(delta, Method, everything())

res3_ls$DMRcate %>% 
  select(-Method) %>% 
  kable()

ProbeLasso

This method shows best performance with adjPvalProbe = 0.05 and meanLassoRadius = 1000. The minDmrSep parameter had no discernable effect, so we left this parameter at its default value.

resPL3_df <- 
  pl_tab %>% 
  filter(adjPval == 0.05) %>% 
  filter(mLassoRad == 1000) %>% 
  filter(minDmrSep == 1000) %>% 
  ungroup() %>% 
  select(-one_of("adjPval", "mLassoRad", "minDmrSep"))
PLtimes_df <- 
  lilyCompTimes_df %>% 
  filter(Method == "ProbeLasso") %>% 
  mutate(time = paste0(round(Mean, 0), " (", round(StdDev, 2), ")")) %>% 
  select(-one_of("Mean", "StdDev")) %>% 
  rename("delta" = "Delta")

res3_ls$ProbeLasso <- 
  resPL3_df %>%
  left_join(PLtimes_df, by = "delta") %>% 
  select(delta, Method, everything())

res3_ls$ProbeLasso %>% 
  select(-Method) %>% 
  kable()

Bumphunter

This method shows best performance with pickCutoffQ = 0.95 and maxGap = 250.

resBump3_df <- 
  bump_tab %>% 
  filter(cutoffQ == 0.95) %>% 
  filter(maxGap == 250) %>% 
  ungroup() %>% 
  select(-one_of("cutoffQ", "maxGap"))
BumpTimes_df <-
  lilyCompTimes_df %>% 
  filter(Method == "Bumphunter") %>% 
  mutate(time = paste0(round(Mean, 0), " (", round(StdDev, 2), ")")) %>% 
  select(-one_of("Mean", "StdDev")) %>% 
  rename("delta" = "Delta")

res3_ls$Bumphunter <- 
  resBump3_df %>%
  left_join(BumpTimes_df, by = "delta") %>% 
  select(delta, Method, everything())

res3_ls$Bumphunter %>% 
  select(-Method) %>% 
  kable()

Comb-p

This method shows best performance with seed = 0.05 and dist = 750.

res3_ls$Comb_p <- 
  comb_tab %>% 
  rename("Method" = "method") %>% 
  ungroup() %>% 
  filter(combSeed == 0.05) %>% 
  filter(combDist == 750) %>% 
  select(-one_of("combSeed", "combDist"))

res3_ls$Comb_p %>% 
  select(-Method) %>% 
  kable()

Save Combined Table

res3_df <-
  res3_ls %>% 
  bind_rows() %>% 
  arrange(delta)

# write_csv(res3_df, path = "../resultsData/Best_params_results.csv")
res3_df %>% 
  kable()

gabrielodom/DMRcomparePaper documentation built on May 25, 2019, 2:52 a.m.