knitr::opts_chunk$set(echo = TRUE)
We have saved the tabulated results files for each method in the DMRcompare
package. We will use the tidyverse
package suite for data management utility. Access these via
library(DMRcompare) library(tidyverse) data("dmrcateRes_df") data("probeLassoRes_df") data("bumphunterRes_df") data("combpRes_df")
These data frames all contain the following common elements:
commonNames_char <- Reduce( intersect, list(colnames(dmrcateRes_df), colnames(probeLassoRes_df), colnames(bumphunterRes_df), colnames(combpRes_df)) ) commonNames_char
The common elements of interest are
FN
), false positives (FP
), true negatives (TN
), true positives (TP
), power, the sample size used to calculate power (nPower
), the area under the precision-recall curve (AuPR
), precision false positives (FPprecis
), precision true positives (TPprecis
), precision, the sample size used to calculate precision (nPrecis
), Matthews Correlation Coefficient (mcc
), and the F1-score (F1
). Because many of these performance metric are related to each other arithmetically, we will reduce our inspection set to AuPR, MCC, and F1.method
), the simulated effect size (delta
), and the random seed to ensure reproducibility (seed
).The method-specific parameters are
lambda
and C
setdiff(colnames(dmrcateRes_df), commonNames_char)
adjPval
, mLassoRad
, and minDmrSep
setdiff(colnames(probeLassoRes_df), commonNames_char)
cutoffQ
and maxGap
setdiff(colnames(bumphunterRes_df), commonNames_char)
combSeed
and combDist
setdiff(colnames(combpRes_df), commonNames_char)
We finally have need for the supplemental tables of all performance metrics for all tested parameter settings under each delta
while averaged over each seed.
This table shows the average performance of the DMRcate method at each $\delta > 0$.
dmrcate_tab <- dmrcateRes_df %>% select(delta, seed, lambda, C, TP, FP, FN, power, precision, AuPR, mcc, F1) %>% filter(delta > 0) %>% group_by(delta, lambda, C) %>% summarise( TP = CalcMeanSD(TP, sigFigsMean = 0), FP = CalcMeanSD(FP, sigFigsMean = 0), FN = CalcMeanSD(FN, sigFigsMean = 0), Pwr = CalcMeanSD(power), Precis = CalcMeanSD(precision), AuPR = CalcMeanSD(AuPR), MCC = CalcMeanSD(mcc), F1 = CalcMeanSD(F1) ) # write_csv(dmrcate_tab, path = "../resultsData/DMRcate_total_results.csv") dmrcate_tab %>% kable()
This table shows the average performance of the ProbeLasso method at each $\delta > 0$.
pl_tab <- probeLassoRes_df %>% select(delta, seed, adjPval, mLassoRad, minDmrSep, TP, FP, FN, power, precision, AuPR, mcc, F1) %>% filter(delta > 0) %>% group_by(delta, adjPval, mLassoRad, minDmrSep) %>% summarise( TP = CalcMeanSD(TP, sigFigsMean = 0), FP = CalcMeanSD(FP, sigFigsMean = 0), FN = CalcMeanSD(FN, sigFigsMean = 0), Pwr = CalcMeanSD(power), Precis = CalcMeanSD(precision), AuPR = CalcMeanSD(AuPR), MCC = CalcMeanSD(mcc), F1 = CalcMeanSD(F1) ) # write_csv(pl_tab, path = "../resultsData/ProbeLasso_total_results.csv") pl_tab %>% kable()
This table shows the average performance of the Bumphunter method at each $\delta > 0$.
bump_tab <- bumphunterRes_df %>% select(delta, seed, cutoffQ, maxGap, TP, FP, FN, power, precision, AuPR, mcc, F1) %>% filter(delta > 0) %>% group_by(delta, cutoffQ, maxGap) %>% summarise( TP = CalcMeanSD(TP, sigFigsMean = 0), FP = CalcMeanSD(FP, sigFigsMean = 0), FN = CalcMeanSD(FN, sigFigsMean = 0), Pwr = CalcMeanSD(power), Precis = CalcMeanSD(precision), AuPR = CalcMeanSD(AuPR), MCC = CalcMeanSD(mcc), F1 = CalcMeanSD(F1) ) # write_csv(bump_tab, path = "../resultsData/Bumphunter_total_results.csv") bump_tab %>% kable()
This table shows the average performance of the Comb-p method at each $\delta > 0$.
comb_tab <- combpRes_df %>% select(delta, method, seed, combSeed, combDist, TP, FP, FN, power, precision, AuPR, mcc, F1, time) %>% filter(delta > 0) %>% group_by(delta, combSeed, combDist) %>% summarise( method = "Comb-p", TP = CalcMeanSD(TP, sigFigsMean = 0), FP = CalcMeanSD(FP, sigFigsMean = 0), FN = CalcMeanSD(FN, sigFigsMean = 0), Pwr = CalcMeanSD(power), Precis = CalcMeanSD(precision), AuPR = CalcMeanSD(AuPR), MCC = CalcMeanSD(mcc), F1 = CalcMeanSD(F1), time = CalcMeanSD(time, sigFigsMean = 0) ) # write_csv(comb_tab, path = "../resultsData/Combp_total_results.csv") comb_tab %>% select(-method) %>% kable()
This method shows best performance with lambda = 500
and C = 5
.
resDMRc3_df <- dmrcate_tab %>% filter(lambda == 500) %>% filter(C == 5) %>% ungroup() %>% select(-one_of("lambda", "C")) DMRcTimes_df <- lilyCompTimes_df %>% filter(Method == "DMRcate") %>% mutate(time = paste0(round(Mean, 0), " (", round(StdDev, 2), ")")) %>% select(-one_of("Mean", "StdDev")) %>% rename("delta" = "Delta") res3_ls$DMRcate <- resDMRc3_df %>% left_join(DMRcTimes_df, by = "delta") %>% select(delta, Method, everything()) res3_ls$DMRcate %>% select(-Method) %>% kable()
This method shows best performance with adjPvalProbe = 0.05
and meanLassoRadius = 1000
. The minDmrSep
parameter had no discernable effect, so we left this parameter at its default value.
resPL3_df <- pl_tab %>% filter(adjPval == 0.05) %>% filter(mLassoRad == 1000) %>% filter(minDmrSep == 1000) %>% ungroup() %>% select(-one_of("adjPval", "mLassoRad", "minDmrSep")) PLtimes_df <- lilyCompTimes_df %>% filter(Method == "ProbeLasso") %>% mutate(time = paste0(round(Mean, 0), " (", round(StdDev, 2), ")")) %>% select(-one_of("Mean", "StdDev")) %>% rename("delta" = "Delta") res3_ls$ProbeLasso <- resPL3_df %>% left_join(PLtimes_df, by = "delta") %>% select(delta, Method, everything()) res3_ls$ProbeLasso %>% select(-Method) %>% kable()
This method shows best performance with pickCutoffQ = 0.95
and maxGap = 250
.
resBump3_df <- bump_tab %>% filter(cutoffQ == 0.95) %>% filter(maxGap == 250) %>% ungroup() %>% select(-one_of("cutoffQ", "maxGap")) BumpTimes_df <- lilyCompTimes_df %>% filter(Method == "Bumphunter") %>% mutate(time = paste0(round(Mean, 0), " (", round(StdDev, 2), ")")) %>% select(-one_of("Mean", "StdDev")) %>% rename("delta" = "Delta") res3_ls$Bumphunter <- resBump3_df %>% left_join(BumpTimes_df, by = "delta") %>% select(delta, Method, everything()) res3_ls$Bumphunter %>% select(-Method) %>% kable()
This method shows best performance with seed = 0.05
and dist = 750
.
res3_ls$Comb_p <- comb_tab %>% rename("Method" = "method") %>% ungroup() %>% filter(combSeed == 0.05) %>% filter(combDist == 750) %>% select(-one_of("combSeed", "combDist")) res3_ls$Comb_p %>% select(-Method) %>% kable()
res3_df <- res3_ls %>% bind_rows() %>% arrange(delta) # write_csv(res3_df, path = "../resultsData/Best_params_results.csv") res3_df %>% kable()
Add the following code to your website.
For more information on customizing the embed code, read Embedding Snippets.