Nothing
Figures for Nested Data Paper"
In NestMRMC: Single Reader Between-Cases AUC Estimator in Nested Data
# Library ----
library(dplyr)
library(iMRMC)
library(mvtnorm)
library(NestMRMC)
library(doParallel)
library(ggplot2)
library(gridExtra)
Figure 3 in the paper--histogram of number of positive and negative ROIs in
balance and unbalance design
## Balance design
sim.config = simu_config(fix_design = T,stream = 1)
data = data_MRMC(sim.config)$data_final
numROI_balance = AUC_per_reader_nest(data)$numROI
## Unbalance design
sim.config = simu_config(correlation_t = 0.8, fix_design = T,stream = 1)
data = data_MRMC(sim.config)$data_final
numROI_unbalance = AUC_per_reader_nest(data)$numROI
numROI_comb = cbind(numROI_balance,numROI_unbalance)
numROI_df_new = data.frame(num_ROI = c(numROI_comb[1,],numROI_comb[2,]),
truth = as.factor(rep(c("pos","neg"),each = 200)),
design = rep(rep(c("balance","unbalance"),each = 100),2)
%>% as.factor())
## histogram
ggplot(numROI_df_new, aes(x = num_ROI,
color = interaction(design,truth),
fill = interaction(design,truth))) +
geom_histogram(alpha = 0.5,bins = 30,
position = "dodge")
Figure 4
Data description for Figure 4
-
The results in these files assume that the truth labels of positive and negative ROIs are fixed across the 10,000 simulations, hence fixed is in the name of the file.
-
In a balanced design each patient has approximately the same number of positive and negative ROIs. In an unbalanced design each patient tends to have unequal number of positive and negative ROIs. Note that for both designs the overall positive and negative ROIs number should be about the same, since the overall probabilities to be positive ROI and negative ROI are 0.5 and 0.5.
-
sim_result_blance_fixed_cov05062021.csv Contains the summarized statistics from 10,000 simulations results in 75 configurations for 100 patients and 10 ROIs per patient. All simulations use the balanced design and fixed truth labels. The 75 configurations are all the possible combinations of 3 AUC values (0.7, 0.8, 0.9), 5 covariance values (0.1, 0.25, 0.5, 0.75, 0.9) and 5 $\rho$ values (0.1, 0.25, 0.5, 0.75, 0.9). The details of parameters settingts in the 75 configurations is available in the Table 1.
-
sim_result_unblance_fixed_cov05062021.csv Contains the summarized statistics from 10,000 simulations for 75 configurations for 100 patients and 10 ROIs per patient. All simulations use the unbalanced design and fixed truth labels.The details of parameters settings in the 75 configurations is available in the Table 1.
-
There are 16 columns in the both files. The detailed information for each column is given at the end of this file.
## Configs
AUC = c(0.7,0.8,0.9)
COV = c(0.1,0.25,0.5,0.75,0.9)
RHO = c(0.1,0.25,0.5,0.75,0.9)
## load data and name the columns
sim_result_blance = read.csv("NestMRMC_simulation_results/sim_result_blance_fixed_cov05062021.csv",header = T)[,-1]
sim_result_unblance = read.csv("NestMRMC_simulation_results/sim_result_unblance_fixed_cov05062021.csv",header = T)[,-1]
sim_result_blance$RHO = rep(rep(RHO,each = 5),3)
sim_result_blance$COV = rep(COV,15)
sim_result_unblance$RHO = rep(rep(RHO,each = 5),3)
sim_result_unblance$COV = rep(COV,15)
## Convert parameter to factor for better visulization
sim_result_blance$RHO = as.factor(rep(rep(RHO,each = 5),3))
sim_result_blance$COV = as.factor(rep(COV,15))
sim_result_blance$TRUE_AUC = as.factor(rep(AUC, each = 25))
sim_result_unblance$RHO = as.factor(rep(rep(RHO,each = 5),3))
sim_result_unblance$COV = as.factor(rep(COV,15))
sim_result_unblance$TRUE_AUC = as.factor(rep(AUC, each = 25))
configs_75 = sim_result_blance[,16:18]
row.names(configs_75) = paste0('Config ', 1:75)
## plot MC truth vs theoretical truth ----
## VAR
p1_MCvsTheo = ggplot(sim_result_blance %>% filter(TRUE_AUC == 0.7),
aes(x = TRUE_VAR_MEAN, y = R1AUC_VAR)) +
geom_point(aes(group = TRUE_AUC, color = COV, shape = RHO), size = 1) +
ggtitle("Balance AUC = 0.7") + ylim(0,0.0017) + xlim(0,0.0017) +
xlab("Theoretical AUC Variance") + ylab("MC AUC Variance") +
geom_abline(intercept = 0, slope = 1,linetype = "dashed", color = "black")+
theme_bw()+ theme(text = element_text(size = 7)) +
guides(colour = guide_legend(override.aes = list(size=1)),
shape = guide_legend(override.aes = list(size=1)))
p2_MCvsTheo = ggplot(sim_result_blance %>% filter(TRUE_AUC == 0.8),
aes(x = TRUE_VAR_MEAN, y = R1AUC_VAR)) +
geom_point(aes(group = TRUE_AUC, color = COV, shape = RHO), size = 1) +
ggtitle("Balance AUC = 0.8") + ylim(0,0.0017) + xlim(0,0.0017) +
xlab("Theoretical AUC Variance") + ylab("MC AUC Variance") +
geom_abline(intercept = 0, slope = 1,linetype = "dashed", color = "black")+
theme_bw()+ theme(text = element_text(size = 7)) +
guides(colour = guide_legend(override.aes = list(size=1)),
shape = guide_legend(override.aes = list(size=1)))
p3_MCvsTheo = ggplot(sim_result_blance %>% filter(TRUE_AUC == 0.9),
aes(x = TRUE_VAR_MEAN, y = R1AUC_VAR)) +
geom_point(aes(group = TRUE_AUC, color = COV, shape = RHO), size = 1) +
ggtitle("Balance AUC = 0.9") + ylim(0,0.0017) + xlim(0,0.0017) +
xlab("Theoretical AUC Variance") + ylab("MC AUC Variance") +
geom_abline(intercept = 0, slope = 1,linetype = "dashed", color = "black")+
theme_bw()+ theme(text = element_text(size = 7)) +
guides(colour = guide_legend(override.aes = list(size=1)),
shape = guide_legend(override.aes = list(size=1)))
p4_MCvsTheo = ggplot(sim_result_unblance %>% filter(TRUE_AUC == 0.7),
aes(x = TRUE_VAR_MEAN, y = R1AUC_VAR)) +
geom_point(aes(group = TRUE_AUC, color = COV, shape = RHO), size = 1) +
ggtitle("Unbalance AUC = 0.7") + ylim(0,0.0017) + xlim(0,0.0017) +
xlab("Theoretical AUC Variance") + ylab("MC AUC Variance") +
geom_abline(intercept = 0, slope = 1,linetype = "dashed", color = "black")+
theme_bw()+ theme(text = element_text(size = 7)) +
guides(colour = guide_legend(override.aes = list(size=1)),
shape = guide_legend(override.aes = list(size=1)))
p5_MCvsTheo = ggplot(sim_result_unblance %>% filter(TRUE_AUC == 0.8),
aes(x = TRUE_VAR_MEAN, y = R1AUC_VAR)) +
geom_point(aes(group = TRUE_AUC, color = COV, shape = RHO), size = 1) +
ggtitle("Unbalance AUC = 0.8") +
ylim(0,0.0017) +
xlim(0,0.0017) +
xlab("Theoretical AUC Variance") +
ylab("MC AUC Variance") +
geom_abline(intercept = 0, slope = 1,linetype = "dashed", color = "black")+
theme_bw()+
theme(text = element_text(size = 7)) +
guides(colour = guide_legend(override.aes = list(size=1)),
shape = guide_legend(override.aes = list(size=1)))
p6_MCvsTheo = ggplot(sim_result_unblance %>% filter(TRUE_AUC == 0.9),
aes(x = TRUE_VAR_MEAN, y = R1AUC_VAR)) +
geom_point(aes(group = TRUE_AUC, color = COV, shape = RHO), size = 1) +
ggtitle("Unbalance AUC = 0.9") +
ylim(0,0.0017) +
xlim(0,0.0017) +
xlab("Theoretical AUC Variance") +
ylab("MC AUC Variance") +
geom_abline(intercept = 0, slope = 1,linetype = "dashed", color = "black") +
theme_bw() +
theme(text = element_text(size = 7)) +
guides(colour = guide_legend(override.aes = list(size=1)),
shape = guide_legend(override.aes = list(size=1)))
grid.arrange(p1_MCvsTheo, p2_MCvsTheo, p3_MCvsTheo,
p4_MCvsTheo, p5_MCvsTheo, p6_MCvsTheo,nrow = 2,ncol = 3)
Figure 5
Data description for Figure 5
-
The results in these files assume that the truth labels of positive and negative ROIs are randomly assigned across the 10,000 simulations, hence unfixed is in the name of the file.
-
In a balanced design each patient has approximately the same number of positive and negative ROIs. In an unbalanced design each patient tends to have unequal number of positive and negative ROIs. Note that for both designs the overall positive and negative ROIs number should be about the same, since the overall probabilities to be positive ROI and negative ROI are 0.5 and 0.5.
-
sim_result_blance_unfixed_cov005182021.csv Contains the summarized statistics from 10,000 simulations results in 75 configurations for 100 patients and 10 ROIs per patient. All simulations use the balanced design and unfixed truth labels. The 75 configurations are all the possible combinations of 3 AUC values (0.7, 0.8, 0.9), 5 covariance values (0.1, 0.25, 0.5, 0.75, 0.9) and 5 $\rho$ values (0.1, 0.25, 0.5, 0.75, 0.9). The details of parameters settingts in the 75 configurations is available in the Table 1.
-
sim_result_unblance_unfixed_cov05182021.csv Contains the summarized statistics from 10,000 simulations for 75 configurations for 100 patients and 10 ROIs per patient. All simulations use the unbalanced design and unfixed truth labels.The details of parameters settings in the 75 configurations is available in the Table 1.
-
There are 16 columns in the both files. The detailed information is provided in the end.
## Load data
sim_result_blance = read.csv("NestMRMC_simulation_results/sim_result_blance_unfixed_cov005182021.csv",header = T)[,-1]
sim_result_unblance = read.csv("NestMRMC_simulation_results/sim_result_unblance_unfixed_cov05182021.csv",header = T)[,-1]
sim_result_blance$RHO = rep(rep(RHO,each = 5),3)
sim_result_blance$COV = rep(COV,15)
sim_result_unblance$RHO = rep(rep(RHO,each = 5),3)
sim_result_unblance$COV = rep(COV,15)
## Convert parameter to factor for better visulization
sim_result_blance$RHO = as.factor(rep(rep(RHO,each = 5),3))
sim_result_blance$COV = as.factor(rep(COV,15))
sim_result_blance$TRUE_AUC = as.factor(rep(AUC, each = 25))
sim_result_blance$TRUE_AUC = rep(AUC, each = 25)
sim_result_unblance$RHO = as.factor(rep(rep(RHO,each = 5),3))
sim_result_unblance$COV = as.factor(rep(COV,15))
sim_result_unblance$TRUE_AUC = as.factor(rep(AUC, each = 25))
sim_result_unblance$TRUE_AUC = rep(AUC, each = 25)
## VAR
p1_MCvsTheo = ggplot(sim_result_blance %>% filter(TRUE_AUC == 0.7),
aes(x = TRUE_VAR_MEAN, y = R1VAR_MEAN)) +
geom_point(aes( color = COV, shape = RHO),size = 1) +
ggtitle("Balance AUC = 0.7") + ylim(0,0.0017) + xlim(0,0.0017) +
xlab("MC truth of AUC VAR") + ylab("MC MEAN of AUC VAR Estimates") +
geom_abline(intercept = 0, slope = 1,linetype = "dashed", color = "black")+
theme_bw()+ theme(text = element_text(size = 7)) +
guides(colour = guide_legend(override.aes = list(size=1)),
shape = guide_legend(override.aes = list(size=1)))
p2_MCvsTheo = ggplot(sim_result_blance %>% filter(TRUE_AUC == 0.8),
aes(x = TRUE_VAR_MEAN, y = R1VAR_MEAN)) +
geom_point(aes( color = COV, shape = RHO),size = 1) +
ggtitle("Balance AUC = 0.8") + ylim(0,0.0017) + xlim(0,0.0017) +
xlab("MC truth of AUC VAR") + ylab("MC MEAN of AUC VAR Estimates") +
geom_abline(intercept = 0, slope = 1,linetype = "dashed", color = "black")+
theme_bw()+ theme(text = element_text(size = 7)) +
guides(colour = guide_legend(override.aes = list(size=1)),
shape = guide_legend(override.aes = list(size=1)))
p3_MCvsTheo = ggplot(sim_result_blance %>% filter(TRUE_AUC == 0.9),
aes(x = TRUE_VAR_MEAN, y = R1VAR_MEAN)) +
geom_point(aes( color = COV, shape = RHO),size = 1) +
ggtitle("Balance AUC = 0.9") + ylim(0,0.0017) + xlim(0,0.0017) +
xlab("MC truth of AUC VAR") + ylab("MC MEAN of AUC VAR Estimates") +
geom_abline(intercept = 0, slope = 1,linetype = "dashed", color = "black")+
theme_bw()+ theme(text = element_text(size = 7)) +
guides(colour = guide_legend(override.aes = list(size=1)),
shape = guide_legend(override.aes = list(size=1)))
p4_MCvsTheo = ggplot(sim_result_unblance %>% filter(TRUE_AUC == 0.7),
aes(x = TRUE_VAR_MEAN, y = R1VAR_MEAN)) +
geom_point(aes( color = COV, shape = RHO),size = 1) +
ggtitle("Unbalance AUC = 0.7") + ylim(0,0.0017) + xlim(0,0.0017) +
xlab("MC truth of AUC VAR") + ylab("MC MEAN of AUC VAR Estimates") +
geom_abline(intercept = 0, slope = 1,linetype = "dashed", color = "black")+
theme_bw()+ theme(text = element_text(size = 7)) +
guides(colour = guide_legend(override.aes = list(size=1)),
shape = guide_legend(override.aes = list(size=1)))
p5_MCvsTheo = ggplot(sim_result_unblance %>% filter(TRUE_AUC == 0.8),
aes(x = TRUE_VAR_MEAN, y = R1VAR_MEAN)) +
geom_point(aes( color = COV, shape = RHO),size = 1) +
ggtitle("Unbalance AUC = 0.8") + ylim(0,0.0017) + xlim(0,0.0017) +
xlab("MC truth of AUC VAR") + ylab("MC MEAN of AUC VAR Estimates") +
geom_abline(intercept = 0, slope = 1,linetype = "dashed", color = "black")+
theme_bw()+ theme(text = element_text(size = 7)) +
guides(colour = guide_legend(override.aes = list(size=1)),
shape = guide_legend(override.aes = list(size=1)))
p6_MCvsTheo = ggplot(sim_result_unblance %>% filter(TRUE_AUC == 0.9),
aes(x = TRUE_VAR_MEAN, y = R1VAR_MEAN)) +
geom_point(aes( color = COV, shape = RHO),size = 1) +
ggtitle("Unbalance AUC = 0.9") + ylim(0,0.0017) + xlim(0,0.0017) +
xlab("MC truth of AUC VAR") + ylab("MC MEAN of AUC VAR Estimates") +
geom_abline(intercept = 0, slope = 1,linetype = "dashed", color = "black")+
theme_bw()+ theme(text = element_text(size = 7)) +
guides(colour = guide_legend(override.aes = list(size=1)),
shape = guide_legend(override.aes = list(size=1)))
grid.arrange(p1_MCvsTheo, p2_MCvsTheo, p3_MCvsTheo,
p4_MCvsTheo, p5_MCvsTheo, p6_MCvsTheo,nrow = 2,ncol = 3)
Figure 6
Data description for Figure 6
-
The results in these files assume that the truth labels of positive and negative ROIs are randomly assigned across the 10,000 simulations, hence unfixed is in the name of the file.
-
In a balanced design each patient has approximately the same number of positive and negative ROIs. In an unbalanced design each patient tends to have unequal number of positive and negative ROIs. Note that for both designs the overall positive and negative ROIs number should be about the same, since the overall probabilities to be positive ROI and negative ROI are 0.5 and 0.5.
-
sim_result_blance_unfixed_cov10042021.csv Contains the summarized statistics from 10,000 simulations results in 75 configurations for 100 patients and 10 ROIs per patient. All simulations use the balanced design and unfixed truth labels. The 75 configurations are all the possible combinations of 3 AUC values (0.7, 0.8, 0.9), 5 covariance values (0.1, 0.25, 0.5, 0.75, 0.9) and 5 $\rho$ values (0.1, 0.25, 0.5, 0.75, 0.9). The details of parameters settingts in the 75 configurations is available in the Table 1.
-
sim_result_unblance_unfixed_cov10042021.csv Contains the summarized statistics from 10,000 simulations for 75 configurations for 100 patients and 10 ROIs per patient. All simulations use the unbalanced design and unfixed truth labels.The details of parameters settingts in the 75 configurations is available in the Table 1.
-
sim_Nancy_blance_unfixed_cov10042021.csv Contains the summarized statistics from 10,000 simulations for 75 configurations for 100 patients and 10 ROIs per patient based on Nancy's method. All simulations use the balanced design and unfixed truth labels.The details of parameters settingts in the 75 configurations is available in the Table 1.
-
sim_Nancy_unblance_unfixed_cov10042021.csv Contains the summarized statistics from 10,000 simulations for 75 configurations for 100 patients and 10 ROIs per patient based on Nancy's method. All simulations use the unbalanced design and unfixed truth labels.The details of parameters settingts in the 75 configurations is available in the Table 1.
-
There are 16 columns in the above files. The detailed information is provided in the end.
## load data ----
sim_result_blance = read.csv("NestMRMC_simulation_results/sim_result_blance_unfixed_cov10042021.csv",header = T)[,-1]
sim_result_unblance = read.csv("NestMRMC_simulation_results/sim_result_unblance_unfixed_cov10042021.csv",header = T)[,-1]
sim_nancy_blance = read.csv("NestMRMC_simulation_results/sim_Nancy_blance_unfixed_cov10042021.csv",header = T)[,-1]
sim_nancy_unblance = read.csv("NestMRMC_simulation_results/sim_Nancy_unblance_unfixed_cov10042021.csv",header = T)[,-1]
AUC = c(0.7,0.8,0.9)%>% as.factor()
COV = c(0.1,0.25,0.5,0.75,0.9)%>% as.factor()
RHO = as.factor(c(0.1,0.25,0.5,0.75,0.9))
## calculate the bias ----
## AUC Bias
sim_result_blance$R1AUC_BIAS = (sim_result_blance$TRUE_AUC - sim_result_blance$R1AUC_MEAN)/sim_result_blance$TRUE_AUC
sim_result_unblance$R1AUC_BIAS = (sim_result_unblance$TRUE_AUC - sim_result_unblance$R1AUC_MEAN)/sim_result_unblance$TRUE_AUC
sim_nancy_blance$R1AUC_BIAS = (sim_nancy_blance$TRUE_AUC - sim_nancy_blance$R1AUC_MEAN)/sim_nancy_blance$TRUE_AUC
sim_nancy_unblance$R1AUC_BIAS = (sim_nancy_unblance$TRUE_AUC - sim_nancy_unblance$R1AUC_MEAN)/sim_nancy_unblance$TRUE_AUC
## VAR Bias
sim_result_blance$R1VAR_BIAS = (sim_result_blance$R1AUC_VAR - sim_result_blance$R1VAR_MEAN)/sim_result_blance$R1AUC_VAR
sim_result_unblance$R1VAR_BIAS = (sim_result_unblance$R1AUC_VAR - sim_result_unblance$R1VAR_MEAN)/sim_result_unblance$R1AUC_VAR
sim_nancy_blance$R1VAR_BIAS = (sim_nancy_blance$R1AUC_VAR - sim_nancy_blance$R1VAR_MEAN)/sim_nancy_blance$R1AUC_VAR
sim_nancy_unblance$R1VAR_BIAS = (sim_nancy_unblance$R1AUC_VAR - sim_nancy_unblance$R1VAR_MEAN)/sim_nancy_unblance$R1AUC_VAR
sim_result_blance$R1VAR_BIAS = (sim_result_blance$R1AUC_VAR - sim_result_blance$R1VAR_MEAN)
sim_result_unblance$R1VAR_BIAS = (sim_result_unblance$R1AUC_VAR - sim_result_unblance$R1VAR_MEAN)
sim_nancy_blance$R1VAR_BIAS = (sim_nancy_blance$R1AUC_VAR - sim_nancy_blance$R1VAR_MEAN)
sim_nancy_unblance$R1VAR_BIAS = (sim_nancy_unblance$R1AUC_VAR - sim_nancy_unblance$R1VAR_MEAN)
## COV Bias
sim_result_blance$R1R2COV_BIAS = (sim_result_blance$R1R2COV_MC - sim_result_blance$R1R2COV_MEAN)/sim_result_blance$R1R2COV_MC
sim_result_unblance$R1R2COV_BIAS = (sim_result_unblance$R1R2COV_MC - sim_result_unblance$R1R2COV_MEAN)/sim_result_unblance$R1R2COV_MC
sim_nancy_blance$R1R2COV_BIAS = (sim_nancy_blance$R1R2COV_MC - sim_nancy_blance$R1R2COV_MEAN)/sim_nancy_blance$R1R2COV_MC
sim_nancy_unblance$R1R2COV_BIAS = (sim_nancy_unblance$R1R2COV_MC - sim_nancy_unblance$R1R2COV_MEAN)/sim_nancy_unblance$R1R2COV_MC
sim_result_blance$R1R2COV_BIAS = (sim_result_blance$R1R2COV_MC - sim_result_blance$R1R2COV_MEAN)
sim_result_unblance$R1R2COV_BIAS = (sim_result_unblance$R1R2COV_MC - sim_result_unblance$R1R2COV_MEAN)
sim_nancy_blance$R1R2COV_BIAS = (sim_nancy_blance$R1R2COV_MC - sim_nancy_blance$R1R2COV_MEAN)
sim_nancy_unblance$R1R2COV_BIAS = (sim_nancy_unblance$R1R2COV_MC - sim_nancy_unblance$R1R2COV_MEAN)
## Bias plot ----
## AUC bias
AUC_bias_combine = data.frame(method = rep(c("CCAUC","Nancy"),each = 150),
design = rep(rep(c("balance","unbalance"),each = 75),2),
bias = c(sim_result_blance$R1AUC_BIAS,sim_result_unblance$R1AUC_BIAS,
sim_nancy_blance$R1AUC_BIAS,sim_nancy_unblance$R1AUC_BIAS))
p1 = ggplot(AUC_bias_combine, aes(x=1:300,y = bias)) + geom_point(aes(color = method, shape = design))+
ggtitle("AUC Relative Bias") + labs(x = "configurations", y = "AUC Relative Bias") +
geom_vline(xintercept = seq(75,225,by = 75), linetype = "dashed", size =0.5) +
theme_bw() + theme(text = element_text(size = 7))
## VAR bias
VAR_bias_combine = data.frame(method = rep(c("CCAUC","Nancy"),each = 150),
design = rep(rep(c("balance","unbalance"),each = 75),2),
AUC = rep(AUC,each = 25),
RHO = rep(rep(rep(RHO,each = 5),3),2),
cov = rep(rep(COV,15),2),
bias = c(sim_result_blance$R1VAR_BIAS,sim_result_unblance$R1VAR_BIAS,
sim_nancy_blance$R1VAR_BIAS,sim_nancy_unblance$R1VAR_BIAS))
p2 = ggplot(VAR_bias_combine%>%filter(method == "CCAUC"), aes(x=1:150,y = bias)) +
geom_point(aes(color = AUC,,shape = RHO))+ labs(x = "configurations", y = "VAR Relative Bias") +
geom_vline(xintercept = seq(75,149,by = 75), linetype = "dashed", size =0.5) +
theme_bw() + theme(text = element_text(size = 7))
## COV bias
COV_bias_combine = data.frame(method = rep(c("CCAUC","Nancy"),each = 150),
design = rep(rep(c("balance","unbalance"),each = 75),2),
AUC = rep(AUC,each = 25),
RHO = rep(rep(rep(RHO,each = 5),3),2),
cov = rep(rep(COV,15),2),
bias = c(sim_result_blance$R1R2COV_BIAS,sim_result_unblance$R1R2COV_BIAS,
sim_nancy_blance$R1R2COV_BIAS,sim_nancy_unblance$R1R2COV_BIAS))
p3 = ggplot(COV_bias_combine%>%filter(method == "CCAUC"), aes(x=1:150,y = bias)) + geom_point(aes(color = AUC,shape = RHO))+
labs(x = "configurations", y = "COV Relative Bias") +
geom_vline(xintercept = seq(75,149,by = 75), linetype = "dashed", size =0.5) +
theme_bw() + theme(text = element_text(size = 7))
#grid.arrange(p1,p2,p3,nrow =1)
## calculate CV ----
## AUC CV
sim_result_blance$R1AUC_CV = sqrt(sim_result_blance$R1AUC_VAR)/sim_result_blance$TRUE_AUC
sim_result_unblance$R1AUC_CV = sqrt(sim_result_unblance$R1AUC_VAR)/sim_result_unblance$TRUE_AUC
sim_nancy_blance$R1AUC_CV = sqrt(sim_nancy_blance$R1AUC_VAR)/sim_nancy_blance$TRUE_AUC
sim_nancy_unblance$R1AUC_CV = sqrt(sim_nancy_unblance$R1AUC_VAR)/sim_nancy_unblance$TRUE_AUC
## VAR CV
sim_result_blance$R1VAR_CV = sqrt(sim_result_blance$R1VAR_VAR)/sim_result_blance$R1AUC_VAR
sim_result_unblance$R1VAR_CV = sqrt(sim_result_unblance$R1VAR_VAR)/sim_result_unblance$R1AUC_VAR
sim_nancy_blance$R1VAR_CV = sqrt(sim_nancy_blance$R1VAR_VAR)/sim_nancy_blance$R1AUC_VAR
sim_nancy_unblance$R1VAR_CV = sqrt(sim_nancy_unblance$R1VAR_VAR)/sim_nancy_unblance$R1AUC_VAR
sim_result_blance$R1VAR_CV = sqrt(sim_result_blance$R1VAR_VAR)
sim_result_unblance$R1VAR_CV = sqrt(sim_result_unblance$R1VAR_VAR)
sim_nancy_blance$R1VAR_CV = sqrt(sim_nancy_blance$R1VAR_VAR)/sim_nancy_blance$R1AUC_VAR
sim_nancy_unblance$R1VAR_CV = sqrt(sim_nancy_unblance$R1VAR_VAR)/sim_nancy_unblance$R1AUC_VAR
## COV CV
sim_result_blance$R1R2COV_CV = sqrt(sim_result_blance$R1R2COV_VAR)/sim_result_blance$R1R2COV_MC
sim_result_unblance$R1R2COV_CV = sqrt(sim_result_unblance$R1R2COV_VAR)/sim_result_unblance$R1R2COV_MC
sim_nancy_blance$R1R2COV_CV = sqrt(sim_nancy_blance$R1R2COV_VAR)/sim_nancy_blance$R1R2COV_MC
sim_nancy_unblance$R1R2COV_CV = sqrt(sim_nancy_unblance$R1R2COV_VAR)/sim_nancy_unblance$R1R2COV_MC
## CV plot ----
## AUC cv
AUC_CV_combine = data.frame(method = rep(c("CCAUC","Nancy"),each = 150),
design = rep(rep(c("balance","unbalance"),each = 75),2),
CV = c(sim_result_blance$R1AUC_CV,sim_result_unblance$R1AUC_CV,
sim_nancy_blance$R1AUC_CV,sim_nancy_unblance$R1AUC_CV))
p4 = ggplot(AUC_CV_combine, aes(x=1:300,y = CV)) + geom_point(aes(color = method, shape = design))+
ggtitle("AUC CV Compare") + labs(x = "configurations", y = "AUC CV") +
geom_vline(xintercept = seq(75,225,by = 75), linetype = "dashed", size =0.5) +
theme_bw() + theme(text = element_text(size = 7))
## VAR CV
VAR_CV_combine = data.frame(method = rep(c("CCAUC","Nancy"),each = 150),
design = rep(rep(c("balance","unbalance"),each = 75),2),
AUC = rep(AUC,each = 25),
RHO = rep(rep(rep(RHO,each = 5),3),2),
cov = rep(rep(COV,15),2),
CV = c(sim_result_blance$R1VAR_CV,sim_result_unblance$R1VAR_CV,
sim_nancy_blance$R1VAR_CV,sim_nancy_unblance$R1VAR_CV))
p5 = ggplot(VAR_CV_combine%>%filter(method == "CCAUC"), aes(x=1:150,y = CV)) + geom_point(aes(color = AUC, shape = RHO))+
labs(x = "configurations", y = "VAR CV") +
geom_vline(xintercept = seq(75,149,by = 75), linetype = "dashed", size =0.5) +
theme_bw() + theme(text = element_text(size = 7))
## COV CV
COV_CV_combine = data.frame(method = rep(c("CCAUC","Nancy"),each = 150),
design = rep(rep(c("balance","unbalance"),each = 75),2),
AUC = rep(AUC,each = 25),
RHO = rep(rep(rep(RHO,each = 5),3),2),
cov = rep(rep(COV,15),2),
CV = c(sim_result_blance$R1R2COV_CV,sim_result_unblance$R1R2COV_CV,
sim_nancy_blance$R1R2COV_CV,sim_nancy_unblance$R1R2COV_CV))
p6 = ggplot(COV_CV_combine%>%filter(method == "CCAUC"), aes(x=1:150,y = CV)) + geom_point(aes(color = AUC, shape = RHO))+
labs(x = "configurations", y = "COV CV") + #ylim(0,2.5) +
geom_vline(xintercept = seq(75,149,by = 75), linetype = "dashed", size =0.5) +
theme_bw() + theme(text = element_text(size = 7))
grid.arrange(p2,p3,p5,p6,nrow = 2)
Columns in the above files
- R1AUC_MEAN: The mean of reader 1 between-cases AUC estimates among 10,000 simulations.
- R2AUC_MEAN: The mean of reader 2 between-cases AUC estimates among 10,000 simulations.
- R1AUC_VAR: The variance of reader 1 between-cases AUC estimates among 10,000 simulations.
- R2AUC_VAR: The variance of reader 2 between-cases AUC estimates among 10,000 simulations.
- R1VAR_MEAN: The mean of reader 1 between-cases AUC variance estimates among 10,000 simulations.
- R2VAR_MEAN: The mean of reader 2 between-cases AUC variance estimates among 10,000 simulations.
- R1VAR_VAR: The variance of reader 1 between-cases AUC variance estimates among 10,000 simulations.
- R2VAR_VAR: The variance of reader 2 between-cases AUC variance estimates among 10,000 simulations.
- R1R2COV_MEAN: The mean of reader 1 and reader 2 between-cases AUC covariance estimates among 10,000 simulations.
- R1R2COV_VAR: The variance of reader 1 and reader 2 between-cases AUC covariance estimates among 10,000 simulations.
- TRUE_VAR_MEAN: The mean of between-cases AUC estimator's theoretical variances among 10,000 simulations.
- TRUE_VAR_VAR: The variance of between-cases AUC estimator's theoretical variances among 10,000 simulations.
- TRUE_COV_MEAN: The mean of between-cases AUC estimator's theoretical covariances among 10,000 simulations.
- TRUE_COV_VAR: The variance of between-cases AUC estimator's theoretical covariances among 10,000 simulations.
- R1R2COV_MC: The covariance between reader 1 and reader 2 between-cases AUC estimates among 10,000 simulations.
- TRUE_AUC: The true AUC values for the simulated data.
75 configurations in the simulation
knitr::kable(configs_75, caption = '75 configurations')
Try the NestMRMC package in your browser
Any scripts or data that you put into this service are public.
NestMRMC documentation built on Oct. 21, 2022, 5:06 p.m.
R Package Documentation
Browse R Packages
We want your feedback!
Note that we can't provide technical support on individual packages. You should contact the package authors for that.
# Library ---- library(dplyr) library(iMRMC) library(mvtnorm) library(NestMRMC) library(doParallel) library(ggplot2) library(gridExtra)
Figure 3 in the paper--histogram of number of positive and negative ROIs in
balance and unbalance design
## Balance design sim.config = simu_config(fix_design = T,stream = 1) data = data_MRMC(sim.config)$data_final numROI_balance = AUC_per_reader_nest(data)$numROI ## Unbalance design sim.config = simu_config(correlation_t = 0.8, fix_design = T,stream = 1) data = data_MRMC(sim.config)$data_final numROI_unbalance = AUC_per_reader_nest(data)$numROI numROI_comb = cbind(numROI_balance,numROI_unbalance) numROI_df_new = data.frame(num_ROI = c(numROI_comb[1,],numROI_comb[2,]), truth = as.factor(rep(c("pos","neg"),each = 200)), design = rep(rep(c("balance","unbalance"),each = 100),2) %>% as.factor()) ## histogram ggplot(numROI_df_new, aes(x = num_ROI, color = interaction(design,truth), fill = interaction(design,truth))) + geom_histogram(alpha = 0.5,bins = 30, position = "dodge")
Figure 4
Data description for Figure 4
-
The results in these files assume that the truth labels of positive and negative ROIs are fixed across the 10,000 simulations, hence fixed is in the name of the file.
-
In a balanced design each patient has approximately the same number of positive and negative ROIs. In an unbalanced design each patient tends to have unequal number of positive and negative ROIs. Note that for both designs the overall positive and negative ROIs number should be about the same, since the overall probabilities to be positive ROI and negative ROI are 0.5 and 0.5.
-
sim_result_blance_fixed_cov05062021.csv Contains the summarized statistics from 10,000 simulations results in 75 configurations for 100 patients and 10 ROIs per patient. All simulations use the balanced design and fixed truth labels. The 75 configurations are all the possible combinations of 3 AUC values (0.7, 0.8, 0.9), 5 covariance values (0.1, 0.25, 0.5, 0.75, 0.9) and 5 $\rho$ values (0.1, 0.25, 0.5, 0.75, 0.9). The details of parameters settingts in the 75 configurations is available in the Table 1.
-
sim_result_unblance_fixed_cov05062021.csv Contains the summarized statistics from 10,000 simulations for 75 configurations for 100 patients and 10 ROIs per patient. All simulations use the unbalanced design and fixed truth labels.The details of parameters settings in the 75 configurations is available in the Table 1.
-
There are 16 columns in the both files. The detailed information for each column is given at the end of this file.
## Configs AUC = c(0.7,0.8,0.9) COV = c(0.1,0.25,0.5,0.75,0.9) RHO = c(0.1,0.25,0.5,0.75,0.9) ## load data and name the columns sim_result_blance = read.csv("NestMRMC_simulation_results/sim_result_blance_fixed_cov05062021.csv",header = T)[,-1] sim_result_unblance = read.csv("NestMRMC_simulation_results/sim_result_unblance_fixed_cov05062021.csv",header = T)[,-1] sim_result_blance$RHO = rep(rep(RHO,each = 5),3) sim_result_blance$COV = rep(COV,15) sim_result_unblance$RHO = rep(rep(RHO,each = 5),3) sim_result_unblance$COV = rep(COV,15) ## Convert parameter to factor for better visulization sim_result_blance$RHO = as.factor(rep(rep(RHO,each = 5),3)) sim_result_blance$COV = as.factor(rep(COV,15)) sim_result_blance$TRUE_AUC = as.factor(rep(AUC, each = 25)) sim_result_unblance$RHO = as.factor(rep(rep(RHO,each = 5),3)) sim_result_unblance$COV = as.factor(rep(COV,15)) sim_result_unblance$TRUE_AUC = as.factor(rep(AUC, each = 25)) configs_75 = sim_result_blance[,16:18] row.names(configs_75) = paste0('Config ', 1:75) ## plot MC truth vs theoretical truth ---- ## VAR p1_MCvsTheo = ggplot(sim_result_blance %>% filter(TRUE_AUC == 0.7), aes(x = TRUE_VAR_MEAN, y = R1AUC_VAR)) + geom_point(aes(group = TRUE_AUC, color = COV, shape = RHO), size = 1) + ggtitle("Balance AUC = 0.7") + ylim(0,0.0017) + xlim(0,0.0017) + xlab("Theoretical AUC Variance") + ylab("MC AUC Variance") + geom_abline(intercept = 0, slope = 1,linetype = "dashed", color = "black")+ theme_bw()+ theme(text = element_text(size = 7)) + guides(colour = guide_legend(override.aes = list(size=1)), shape = guide_legend(override.aes = list(size=1))) p2_MCvsTheo = ggplot(sim_result_blance %>% filter(TRUE_AUC == 0.8), aes(x = TRUE_VAR_MEAN, y = R1AUC_VAR)) + geom_point(aes(group = TRUE_AUC, color = COV, shape = RHO), size = 1) + ggtitle("Balance AUC = 0.8") + ylim(0,0.0017) + xlim(0,0.0017) + xlab("Theoretical AUC Variance") + ylab("MC AUC Variance") + geom_abline(intercept = 0, slope = 1,linetype = "dashed", color = "black")+ theme_bw()+ theme(text = element_text(size = 7)) + guides(colour = guide_legend(override.aes = list(size=1)), shape = guide_legend(override.aes = list(size=1))) p3_MCvsTheo = ggplot(sim_result_blance %>% filter(TRUE_AUC == 0.9), aes(x = TRUE_VAR_MEAN, y = R1AUC_VAR)) + geom_point(aes(group = TRUE_AUC, color = COV, shape = RHO), size = 1) + ggtitle("Balance AUC = 0.9") + ylim(0,0.0017) + xlim(0,0.0017) + xlab("Theoretical AUC Variance") + ylab("MC AUC Variance") + geom_abline(intercept = 0, slope = 1,linetype = "dashed", color = "black")+ theme_bw()+ theme(text = element_text(size = 7)) + guides(colour = guide_legend(override.aes = list(size=1)), shape = guide_legend(override.aes = list(size=1))) p4_MCvsTheo = ggplot(sim_result_unblance %>% filter(TRUE_AUC == 0.7), aes(x = TRUE_VAR_MEAN, y = R1AUC_VAR)) + geom_point(aes(group = TRUE_AUC, color = COV, shape = RHO), size = 1) + ggtitle("Unbalance AUC = 0.7") + ylim(0,0.0017) + xlim(0,0.0017) + xlab("Theoretical AUC Variance") + ylab("MC AUC Variance") + geom_abline(intercept = 0, slope = 1,linetype = "dashed", color = "black")+ theme_bw()+ theme(text = element_text(size = 7)) + guides(colour = guide_legend(override.aes = list(size=1)), shape = guide_legend(override.aes = list(size=1))) p5_MCvsTheo = ggplot(sim_result_unblance %>% filter(TRUE_AUC == 0.8), aes(x = TRUE_VAR_MEAN, y = R1AUC_VAR)) + geom_point(aes(group = TRUE_AUC, color = COV, shape = RHO), size = 1) + ggtitle("Unbalance AUC = 0.8") + ylim(0,0.0017) + xlim(0,0.0017) + xlab("Theoretical AUC Variance") + ylab("MC AUC Variance") + geom_abline(intercept = 0, slope = 1,linetype = "dashed", color = "black")+ theme_bw()+ theme(text = element_text(size = 7)) + guides(colour = guide_legend(override.aes = list(size=1)), shape = guide_legend(override.aes = list(size=1))) p6_MCvsTheo = ggplot(sim_result_unblance %>% filter(TRUE_AUC == 0.9), aes(x = TRUE_VAR_MEAN, y = R1AUC_VAR)) + geom_point(aes(group = TRUE_AUC, color = COV, shape = RHO), size = 1) + ggtitle("Unbalance AUC = 0.9") + ylim(0,0.0017) + xlim(0,0.0017) + xlab("Theoretical AUC Variance") + ylab("MC AUC Variance") + geom_abline(intercept = 0, slope = 1,linetype = "dashed", color = "black") + theme_bw() + theme(text = element_text(size = 7)) + guides(colour = guide_legend(override.aes = list(size=1)), shape = guide_legend(override.aes = list(size=1))) grid.arrange(p1_MCvsTheo, p2_MCvsTheo, p3_MCvsTheo, p4_MCvsTheo, p5_MCvsTheo, p6_MCvsTheo,nrow = 2,ncol = 3)
Figure 5
Data description for Figure 5
-
The results in these files assume that the truth labels of positive and negative ROIs are randomly assigned across the 10,000 simulations, hence unfixed is in the name of the file.
-
In a balanced design each patient has approximately the same number of positive and negative ROIs. In an unbalanced design each patient tends to have unequal number of positive and negative ROIs. Note that for both designs the overall positive and negative ROIs number should be about the same, since the overall probabilities to be positive ROI and negative ROI are 0.5 and 0.5.
-
sim_result_blance_unfixed_cov005182021.csv Contains the summarized statistics from 10,000 simulations results in 75 configurations for 100 patients and 10 ROIs per patient. All simulations use the balanced design and unfixed truth labels. The 75 configurations are all the possible combinations of 3 AUC values (0.7, 0.8, 0.9), 5 covariance values (0.1, 0.25, 0.5, 0.75, 0.9) and 5 $\rho$ values (0.1, 0.25, 0.5, 0.75, 0.9). The details of parameters settingts in the 75 configurations is available in the Table 1.
-
sim_result_unblance_unfixed_cov05182021.csv Contains the summarized statistics from 10,000 simulations for 75 configurations for 100 patients and 10 ROIs per patient. All simulations use the unbalanced design and unfixed truth labels.The details of parameters settings in the 75 configurations is available in the Table 1.
-
There are 16 columns in the both files. The detailed information is provided in the end.
## Load data sim_result_blance = read.csv("NestMRMC_simulation_results/sim_result_blance_unfixed_cov005182021.csv",header = T)[,-1] sim_result_unblance = read.csv("NestMRMC_simulation_results/sim_result_unblance_unfixed_cov05182021.csv",header = T)[,-1] sim_result_blance$RHO = rep(rep(RHO,each = 5),3) sim_result_blance$COV = rep(COV,15) sim_result_unblance$RHO = rep(rep(RHO,each = 5),3) sim_result_unblance$COV = rep(COV,15) ## Convert parameter to factor for better visulization sim_result_blance$RHO = as.factor(rep(rep(RHO,each = 5),3)) sim_result_blance$COV = as.factor(rep(COV,15)) sim_result_blance$TRUE_AUC = as.factor(rep(AUC, each = 25)) sim_result_blance$TRUE_AUC = rep(AUC, each = 25) sim_result_unblance$RHO = as.factor(rep(rep(RHO,each = 5),3)) sim_result_unblance$COV = as.factor(rep(COV,15)) sim_result_unblance$TRUE_AUC = as.factor(rep(AUC, each = 25)) sim_result_unblance$TRUE_AUC = rep(AUC, each = 25) ## VAR p1_MCvsTheo = ggplot(sim_result_blance %>% filter(TRUE_AUC == 0.7), aes(x = TRUE_VAR_MEAN, y = R1VAR_MEAN)) + geom_point(aes( color = COV, shape = RHO),size = 1) + ggtitle("Balance AUC = 0.7") + ylim(0,0.0017) + xlim(0,0.0017) + xlab("MC truth of AUC VAR") + ylab("MC MEAN of AUC VAR Estimates") + geom_abline(intercept = 0, slope = 1,linetype = "dashed", color = "black")+ theme_bw()+ theme(text = element_text(size = 7)) + guides(colour = guide_legend(override.aes = list(size=1)), shape = guide_legend(override.aes = list(size=1))) p2_MCvsTheo = ggplot(sim_result_blance %>% filter(TRUE_AUC == 0.8), aes(x = TRUE_VAR_MEAN, y = R1VAR_MEAN)) + geom_point(aes( color = COV, shape = RHO),size = 1) + ggtitle("Balance AUC = 0.8") + ylim(0,0.0017) + xlim(0,0.0017) + xlab("MC truth of AUC VAR") + ylab("MC MEAN of AUC VAR Estimates") + geom_abline(intercept = 0, slope = 1,linetype = "dashed", color = "black")+ theme_bw()+ theme(text = element_text(size = 7)) + guides(colour = guide_legend(override.aes = list(size=1)), shape = guide_legend(override.aes = list(size=1))) p3_MCvsTheo = ggplot(sim_result_blance %>% filter(TRUE_AUC == 0.9), aes(x = TRUE_VAR_MEAN, y = R1VAR_MEAN)) + geom_point(aes( color = COV, shape = RHO),size = 1) + ggtitle("Balance AUC = 0.9") + ylim(0,0.0017) + xlim(0,0.0017) + xlab("MC truth of AUC VAR") + ylab("MC MEAN of AUC VAR Estimates") + geom_abline(intercept = 0, slope = 1,linetype = "dashed", color = "black")+ theme_bw()+ theme(text = element_text(size = 7)) + guides(colour = guide_legend(override.aes = list(size=1)), shape = guide_legend(override.aes = list(size=1))) p4_MCvsTheo = ggplot(sim_result_unblance %>% filter(TRUE_AUC == 0.7), aes(x = TRUE_VAR_MEAN, y = R1VAR_MEAN)) + geom_point(aes( color = COV, shape = RHO),size = 1) + ggtitle("Unbalance AUC = 0.7") + ylim(0,0.0017) + xlim(0,0.0017) + xlab("MC truth of AUC VAR") + ylab("MC MEAN of AUC VAR Estimates") + geom_abline(intercept = 0, slope = 1,linetype = "dashed", color = "black")+ theme_bw()+ theme(text = element_text(size = 7)) + guides(colour = guide_legend(override.aes = list(size=1)), shape = guide_legend(override.aes = list(size=1))) p5_MCvsTheo = ggplot(sim_result_unblance %>% filter(TRUE_AUC == 0.8), aes(x = TRUE_VAR_MEAN, y = R1VAR_MEAN)) + geom_point(aes( color = COV, shape = RHO),size = 1) + ggtitle("Unbalance AUC = 0.8") + ylim(0,0.0017) + xlim(0,0.0017) + xlab("MC truth of AUC VAR") + ylab("MC MEAN of AUC VAR Estimates") + geom_abline(intercept = 0, slope = 1,linetype = "dashed", color = "black")+ theme_bw()+ theme(text = element_text(size = 7)) + guides(colour = guide_legend(override.aes = list(size=1)), shape = guide_legend(override.aes = list(size=1))) p6_MCvsTheo = ggplot(sim_result_unblance %>% filter(TRUE_AUC == 0.9), aes(x = TRUE_VAR_MEAN, y = R1VAR_MEAN)) + geom_point(aes( color = COV, shape = RHO),size = 1) + ggtitle("Unbalance AUC = 0.9") + ylim(0,0.0017) + xlim(0,0.0017) + xlab("MC truth of AUC VAR") + ylab("MC MEAN of AUC VAR Estimates") + geom_abline(intercept = 0, slope = 1,linetype = "dashed", color = "black")+ theme_bw()+ theme(text = element_text(size = 7)) + guides(colour = guide_legend(override.aes = list(size=1)), shape = guide_legend(override.aes = list(size=1))) grid.arrange(p1_MCvsTheo, p2_MCvsTheo, p3_MCvsTheo, p4_MCvsTheo, p5_MCvsTheo, p6_MCvsTheo,nrow = 2,ncol = 3)
Figure 6
Data description for Figure 6
-
The results in these files assume that the truth labels of positive and negative ROIs are randomly assigned across the 10,000 simulations, hence unfixed is in the name of the file.
-
In a balanced design each patient has approximately the same number of positive and negative ROIs. In an unbalanced design each patient tends to have unequal number of positive and negative ROIs. Note that for both designs the overall positive and negative ROIs number should be about the same, since the overall probabilities to be positive ROI and negative ROI are 0.5 and 0.5.
-
sim_result_blance_unfixed_cov10042021.csv Contains the summarized statistics from 10,000 simulations results in 75 configurations for 100 patients and 10 ROIs per patient. All simulations use the balanced design and unfixed truth labels. The 75 configurations are all the possible combinations of 3 AUC values (0.7, 0.8, 0.9), 5 covariance values (0.1, 0.25, 0.5, 0.75, 0.9) and 5 $\rho$ values (0.1, 0.25, 0.5, 0.75, 0.9). The details of parameters settingts in the 75 configurations is available in the Table 1.
-
sim_result_unblance_unfixed_cov10042021.csv Contains the summarized statistics from 10,000 simulations for 75 configurations for 100 patients and 10 ROIs per patient. All simulations use the unbalanced design and unfixed truth labels.The details of parameters settingts in the 75 configurations is available in the Table 1.
-
sim_Nancy_blance_unfixed_cov10042021.csv Contains the summarized statistics from 10,000 simulations for 75 configurations for 100 patients and 10 ROIs per patient based on Nancy's method. All simulations use the balanced design and unfixed truth labels.The details of parameters settingts in the 75 configurations is available in the Table 1.
-
sim_Nancy_unblance_unfixed_cov10042021.csv Contains the summarized statistics from 10,000 simulations for 75 configurations for 100 patients and 10 ROIs per patient based on Nancy's method. All simulations use the unbalanced design and unfixed truth labels.The details of parameters settingts in the 75 configurations is available in the Table 1.
-
There are 16 columns in the above files. The detailed information is provided in the end.
## load data ---- sim_result_blance = read.csv("NestMRMC_simulation_results/sim_result_blance_unfixed_cov10042021.csv",header = T)[,-1] sim_result_unblance = read.csv("NestMRMC_simulation_results/sim_result_unblance_unfixed_cov10042021.csv",header = T)[,-1] sim_nancy_blance = read.csv("NestMRMC_simulation_results/sim_Nancy_blance_unfixed_cov10042021.csv",header = T)[,-1] sim_nancy_unblance = read.csv("NestMRMC_simulation_results/sim_Nancy_unblance_unfixed_cov10042021.csv",header = T)[,-1] AUC = c(0.7,0.8,0.9)%>% as.factor() COV = c(0.1,0.25,0.5,0.75,0.9)%>% as.factor() RHO = as.factor(c(0.1,0.25,0.5,0.75,0.9)) ## calculate the bias ---- ## AUC Bias sim_result_blance$R1AUC_BIAS = (sim_result_blance$TRUE_AUC - sim_result_blance$R1AUC_MEAN)/sim_result_blance$TRUE_AUC sim_result_unblance$R1AUC_BIAS = (sim_result_unblance$TRUE_AUC - sim_result_unblance$R1AUC_MEAN)/sim_result_unblance$TRUE_AUC sim_nancy_blance$R1AUC_BIAS = (sim_nancy_blance$TRUE_AUC - sim_nancy_blance$R1AUC_MEAN)/sim_nancy_blance$TRUE_AUC sim_nancy_unblance$R1AUC_BIAS = (sim_nancy_unblance$TRUE_AUC - sim_nancy_unblance$R1AUC_MEAN)/sim_nancy_unblance$TRUE_AUC ## VAR Bias sim_result_blance$R1VAR_BIAS = (sim_result_blance$R1AUC_VAR - sim_result_blance$R1VAR_MEAN)/sim_result_blance$R1AUC_VAR sim_result_unblance$R1VAR_BIAS = (sim_result_unblance$R1AUC_VAR - sim_result_unblance$R1VAR_MEAN)/sim_result_unblance$R1AUC_VAR sim_nancy_blance$R1VAR_BIAS = (sim_nancy_blance$R1AUC_VAR - sim_nancy_blance$R1VAR_MEAN)/sim_nancy_blance$R1AUC_VAR sim_nancy_unblance$R1VAR_BIAS = (sim_nancy_unblance$R1AUC_VAR - sim_nancy_unblance$R1VAR_MEAN)/sim_nancy_unblance$R1AUC_VAR sim_result_blance$R1VAR_BIAS = (sim_result_blance$R1AUC_VAR - sim_result_blance$R1VAR_MEAN) sim_result_unblance$R1VAR_BIAS = (sim_result_unblance$R1AUC_VAR - sim_result_unblance$R1VAR_MEAN) sim_nancy_blance$R1VAR_BIAS = (sim_nancy_blance$R1AUC_VAR - sim_nancy_blance$R1VAR_MEAN) sim_nancy_unblance$R1VAR_BIAS = (sim_nancy_unblance$R1AUC_VAR - sim_nancy_unblance$R1VAR_MEAN) ## COV Bias sim_result_blance$R1R2COV_BIAS = (sim_result_blance$R1R2COV_MC - sim_result_blance$R1R2COV_MEAN)/sim_result_blance$R1R2COV_MC sim_result_unblance$R1R2COV_BIAS = (sim_result_unblance$R1R2COV_MC - sim_result_unblance$R1R2COV_MEAN)/sim_result_unblance$R1R2COV_MC sim_nancy_blance$R1R2COV_BIAS = (sim_nancy_blance$R1R2COV_MC - sim_nancy_blance$R1R2COV_MEAN)/sim_nancy_blance$R1R2COV_MC sim_nancy_unblance$R1R2COV_BIAS = (sim_nancy_unblance$R1R2COV_MC - sim_nancy_unblance$R1R2COV_MEAN)/sim_nancy_unblance$R1R2COV_MC sim_result_blance$R1R2COV_BIAS = (sim_result_blance$R1R2COV_MC - sim_result_blance$R1R2COV_MEAN) sim_result_unblance$R1R2COV_BIAS = (sim_result_unblance$R1R2COV_MC - sim_result_unblance$R1R2COV_MEAN) sim_nancy_blance$R1R2COV_BIAS = (sim_nancy_blance$R1R2COV_MC - sim_nancy_blance$R1R2COV_MEAN) sim_nancy_unblance$R1R2COV_BIAS = (sim_nancy_unblance$R1R2COV_MC - sim_nancy_unblance$R1R2COV_MEAN) ## Bias plot ---- ## AUC bias AUC_bias_combine = data.frame(method = rep(c("CCAUC","Nancy"),each = 150), design = rep(rep(c("balance","unbalance"),each = 75),2), bias = c(sim_result_blance$R1AUC_BIAS,sim_result_unblance$R1AUC_BIAS, sim_nancy_blance$R1AUC_BIAS,sim_nancy_unblance$R1AUC_BIAS)) p1 = ggplot(AUC_bias_combine, aes(x=1:300,y = bias)) + geom_point(aes(color = method, shape = design))+ ggtitle("AUC Relative Bias") + labs(x = "configurations", y = "AUC Relative Bias") + geom_vline(xintercept = seq(75,225,by = 75), linetype = "dashed", size =0.5) + theme_bw() + theme(text = element_text(size = 7)) ## VAR bias VAR_bias_combine = data.frame(method = rep(c("CCAUC","Nancy"),each = 150), design = rep(rep(c("balance","unbalance"),each = 75),2), AUC = rep(AUC,each = 25), RHO = rep(rep(rep(RHO,each = 5),3),2), cov = rep(rep(COV,15),2), bias = c(sim_result_blance$R1VAR_BIAS,sim_result_unblance$R1VAR_BIAS, sim_nancy_blance$R1VAR_BIAS,sim_nancy_unblance$R1VAR_BIAS)) p2 = ggplot(VAR_bias_combine%>%filter(method == "CCAUC"), aes(x=1:150,y = bias)) + geom_point(aes(color = AUC,,shape = RHO))+ labs(x = "configurations", y = "VAR Relative Bias") + geom_vline(xintercept = seq(75,149,by = 75), linetype = "dashed", size =0.5) + theme_bw() + theme(text = element_text(size = 7)) ## COV bias COV_bias_combine = data.frame(method = rep(c("CCAUC","Nancy"),each = 150), design = rep(rep(c("balance","unbalance"),each = 75),2), AUC = rep(AUC,each = 25), RHO = rep(rep(rep(RHO,each = 5),3),2), cov = rep(rep(COV,15),2), bias = c(sim_result_blance$R1R2COV_BIAS,sim_result_unblance$R1R2COV_BIAS, sim_nancy_blance$R1R2COV_BIAS,sim_nancy_unblance$R1R2COV_BIAS)) p3 = ggplot(COV_bias_combine%>%filter(method == "CCAUC"), aes(x=1:150,y = bias)) + geom_point(aes(color = AUC,shape = RHO))+ labs(x = "configurations", y = "COV Relative Bias") + geom_vline(xintercept = seq(75,149,by = 75), linetype = "dashed", size =0.5) + theme_bw() + theme(text = element_text(size = 7)) #grid.arrange(p1,p2,p3,nrow =1) ## calculate CV ---- ## AUC CV sim_result_blance$R1AUC_CV = sqrt(sim_result_blance$R1AUC_VAR)/sim_result_blance$TRUE_AUC sim_result_unblance$R1AUC_CV = sqrt(sim_result_unblance$R1AUC_VAR)/sim_result_unblance$TRUE_AUC sim_nancy_blance$R1AUC_CV = sqrt(sim_nancy_blance$R1AUC_VAR)/sim_nancy_blance$TRUE_AUC sim_nancy_unblance$R1AUC_CV = sqrt(sim_nancy_unblance$R1AUC_VAR)/sim_nancy_unblance$TRUE_AUC ## VAR CV sim_result_blance$R1VAR_CV = sqrt(sim_result_blance$R1VAR_VAR)/sim_result_blance$R1AUC_VAR sim_result_unblance$R1VAR_CV = sqrt(sim_result_unblance$R1VAR_VAR)/sim_result_unblance$R1AUC_VAR sim_nancy_blance$R1VAR_CV = sqrt(sim_nancy_blance$R1VAR_VAR)/sim_nancy_blance$R1AUC_VAR sim_nancy_unblance$R1VAR_CV = sqrt(sim_nancy_unblance$R1VAR_VAR)/sim_nancy_unblance$R1AUC_VAR sim_result_blance$R1VAR_CV = sqrt(sim_result_blance$R1VAR_VAR) sim_result_unblance$R1VAR_CV = sqrt(sim_result_unblance$R1VAR_VAR) sim_nancy_blance$R1VAR_CV = sqrt(sim_nancy_blance$R1VAR_VAR)/sim_nancy_blance$R1AUC_VAR sim_nancy_unblance$R1VAR_CV = sqrt(sim_nancy_unblance$R1VAR_VAR)/sim_nancy_unblance$R1AUC_VAR ## COV CV sim_result_blance$R1R2COV_CV = sqrt(sim_result_blance$R1R2COV_VAR)/sim_result_blance$R1R2COV_MC sim_result_unblance$R1R2COV_CV = sqrt(sim_result_unblance$R1R2COV_VAR)/sim_result_unblance$R1R2COV_MC sim_nancy_blance$R1R2COV_CV = sqrt(sim_nancy_blance$R1R2COV_VAR)/sim_nancy_blance$R1R2COV_MC sim_nancy_unblance$R1R2COV_CV = sqrt(sim_nancy_unblance$R1R2COV_VAR)/sim_nancy_unblance$R1R2COV_MC ## CV plot ---- ## AUC cv AUC_CV_combine = data.frame(method = rep(c("CCAUC","Nancy"),each = 150), design = rep(rep(c("balance","unbalance"),each = 75),2), CV = c(sim_result_blance$R1AUC_CV,sim_result_unblance$R1AUC_CV, sim_nancy_blance$R1AUC_CV,sim_nancy_unblance$R1AUC_CV)) p4 = ggplot(AUC_CV_combine, aes(x=1:300,y = CV)) + geom_point(aes(color = method, shape = design))+ ggtitle("AUC CV Compare") + labs(x = "configurations", y = "AUC CV") + geom_vline(xintercept = seq(75,225,by = 75), linetype = "dashed", size =0.5) + theme_bw() + theme(text = element_text(size = 7)) ## VAR CV VAR_CV_combine = data.frame(method = rep(c("CCAUC","Nancy"),each = 150), design = rep(rep(c("balance","unbalance"),each = 75),2), AUC = rep(AUC,each = 25), RHO = rep(rep(rep(RHO,each = 5),3),2), cov = rep(rep(COV,15),2), CV = c(sim_result_blance$R1VAR_CV,sim_result_unblance$R1VAR_CV, sim_nancy_blance$R1VAR_CV,sim_nancy_unblance$R1VAR_CV)) p5 = ggplot(VAR_CV_combine%>%filter(method == "CCAUC"), aes(x=1:150,y = CV)) + geom_point(aes(color = AUC, shape = RHO))+ labs(x = "configurations", y = "VAR CV") + geom_vline(xintercept = seq(75,149,by = 75), linetype = "dashed", size =0.5) + theme_bw() + theme(text = element_text(size = 7)) ## COV CV COV_CV_combine = data.frame(method = rep(c("CCAUC","Nancy"),each = 150), design = rep(rep(c("balance","unbalance"),each = 75),2), AUC = rep(AUC,each = 25), RHO = rep(rep(rep(RHO,each = 5),3),2), cov = rep(rep(COV,15),2), CV = c(sim_result_blance$R1R2COV_CV,sim_result_unblance$R1R2COV_CV, sim_nancy_blance$R1R2COV_CV,sim_nancy_unblance$R1R2COV_CV)) p6 = ggplot(COV_CV_combine%>%filter(method == "CCAUC"), aes(x=1:150,y = CV)) + geom_point(aes(color = AUC, shape = RHO))+ labs(x = "configurations", y = "COV CV") + #ylim(0,2.5) + geom_vline(xintercept = seq(75,149,by = 75), linetype = "dashed", size =0.5) + theme_bw() + theme(text = element_text(size = 7)) grid.arrange(p2,p3,p5,p6,nrow = 2)
Columns in the above files
- R1AUC_MEAN: The mean of reader 1 between-cases AUC estimates among 10,000 simulations.
- R2AUC_MEAN: The mean of reader 2 between-cases AUC estimates among 10,000 simulations.
- R1AUC_VAR: The variance of reader 1 between-cases AUC estimates among 10,000 simulations.
- R2AUC_VAR: The variance of reader 2 between-cases AUC estimates among 10,000 simulations.
- R1VAR_MEAN: The mean of reader 1 between-cases AUC variance estimates among 10,000 simulations.
- R2VAR_MEAN: The mean of reader 2 between-cases AUC variance estimates among 10,000 simulations.
- R1VAR_VAR: The variance of reader 1 between-cases AUC variance estimates among 10,000 simulations.
- R2VAR_VAR: The variance of reader 2 between-cases AUC variance estimates among 10,000 simulations.
- R1R2COV_MEAN: The mean of reader 1 and reader 2 between-cases AUC covariance estimates among 10,000 simulations.
- R1R2COV_VAR: The variance of reader 1 and reader 2 between-cases AUC covariance estimates among 10,000 simulations.
- TRUE_VAR_MEAN: The mean of between-cases AUC estimator's theoretical variances among 10,000 simulations.
- TRUE_VAR_VAR: The variance of between-cases AUC estimator's theoretical variances among 10,000 simulations.
- TRUE_COV_MEAN: The mean of between-cases AUC estimator's theoretical covariances among 10,000 simulations.
- TRUE_COV_VAR: The variance of between-cases AUC estimator's theoretical covariances among 10,000 simulations.
- R1R2COV_MC: The covariance between reader 1 and reader 2 between-cases AUC estimates among 10,000 simulations.
- TRUE_AUC: The true AUC values for the simulated data.
75 configurations in the simulation
knitr::kable(configs_75, caption = '75 configurations')
Try the NestMRMC package in your browser
Any scripts or data that you put into this service are public.
R Package Documentation
Browse R Packages
We want your feedback!
Note that we can't provide technical support on individual packages. You should contact the package authors for that.
Embedding an R snippet on your website
Add the following code to your website.
For more information on customizing the embed code, read Embedding Snippets.