R/tables_generator.R

In schemaanalyst/directedRandom-R-pkg: Directed Random

#' FUNCTION: table_generator_coverage
#'
#' Generates a latex table or a data frame for coverage table with effect size and U test.
#' @param d Data frame of analysis
#' @param rtrn Latex (tex) or a data frame (data)
#' @param m Results shown as median or mean
#' @return A A12 effect size and U-test of coverages compared pair wise
#' @importFrom magrittr %>%
#' @export
table_generator_coverage <- function(d, rtrn = "tex", m = "median") {
  # Arrange dataframe by case study
  d <- d %>% dplyr::arrange(casestudy)
  # Store the dataframe into another var
  d1 <- d
  # generate a DF for mean or median
  if (m == "mean") {
    d <- d %>% dplyr::select(dbms, casestudy, datagenerator, coverage, randomseed) %>% dplyr::group_by(dbms, casestudy, datagenerator) %>% dplyr::summarise(coverage = format(round((mean(coverage)), 1), nsmall = 1))
  } else {
    d <- d %>% dplyr::select(dbms, casestudy, datagenerator, coverage, randomseed) %>% dplyr::group_by(dbms, casestudy, datagenerator) %>% dplyr::summarise(coverage = format(round((median(coverage)), 1), nsmall = 1))
  }
  # filp the data frame
  d <- reshape2::dcast(d, casestudy ~ dbms + datagenerator, value.var=c("coverage"))
  # get header
  a1 <- d[1]
  # Split by DBMS
  d2 <- d[2:13]
  d <- d2[ , order(names(d2))]
  c <- d[1:4]
  c <- c[c(3,1,2,4)]
  a <- d[5:8]
  a <- a[c(3,1,2,4)]
  b <- d[9:12]
  b <- b[c(3,1,2,4)]
  # change the schemas from fectors to char
  a1$casestudy <- as.character(a1$casestudy)
  # get nunber of rows and itrate through them
  numberOfRows <- nrow(d)
  for (i in 1:numberOfRows) {
    schema <- a1[i,]
    # get each generators
    dr <- d1 %>% dplyr::filter(casestudy == schema, datagenerator == "directedRandom")
    avm <- d1 %>% dplyr::filter(casestudy == schema, datagenerator == "avs")
    avmd <- d1 %>% dplyr::filter(casestudy == schema, datagenerator == "avsDefaults")
    rand <- d1 %>% dplyr::filter(casestudy == schema, datagenerator == "random")
    #a[i,] = ifelse(min(as.numeric(a[i,])) == as.numeric(a[i,]), paste("\\textbf{", a[i,], "}", sep = ""), as.numeric(a[i,]))

    # Effect size for PSQL
    postgres_avm <- directedRandomR::effectsize_accurate((dr %>% dplyr::filter(dbms == "Postgres"))$coverage, (avm %>% dplyr::filter(dbms == "Postgres"))$coverage)$size
    postgres_avmd <- directedRandomR::effectsize_accurate((dr %>% dplyr::filter(dbms == "Postgres"))$coverage, (avmd %>% dplyr::filter(dbms == "Postgres"))$coverage)$size
    postgres_rand <- directedRandomR::effectsize_accurate((dr %>% dplyr::filter(dbms == "Postgres"))$coverage, (rand %>% dplyr::filter(dbms == "Postgres"))$coverage)$size

    # get coverage
    dr_coverage <- (dr %>% dplyr::filter(dbms == "Postgres"))$coverage
    avmr_coverage <- (avm %>% dplyr::filter(dbms == "Postgres"))$coverage

    # U-test AVMR vs DR
    p1 <- wilcox.test(dr_coverage, avmr_coverage, alternative = "greater", exact = FALSE)$p.value <= 0.01
    p2 <- wilcox.test(dr_coverage, avmr_coverage, alternative = "less", exact = FALSE)$p.value <= 0.01

    # Check one-sided test
    if (p1 == TRUE & p2 == FALSE) {
      a[i,2] = paste("\\textbf{",a[i,2],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      a[i,2] = paste("\\textit{",a[i,2],"}", sep = "")
    } else {
    }
    # check effect
    if (postgres_avm == "large") {
      a[i,2] = paste("$^{\\ast\\ast\\ast}$",a[i,2], sep = "")
    } else if (postgres_avm == "medium") {
      a[i,2] = paste("$^{\\ast\\ast}$",a[i,2], sep = "")
    } else if (postgres_avm == "small") {
      a[i,2] = paste("$^{\\ast}$",a[i,2], sep = "")
    } else {

    }

    # get coverage
    avmd_coverage <- (avmd %>% dplyr::filter(dbms == "Postgres"))$coverage
    # U-test AVMD vs DR
    p1 <- wilcox.test(dr_coverage, avmd_coverage, alternative = "greater", exact = FALSE)$p.value <= 0.01
    p2 <- wilcox.test(dr_coverage, avmd_coverage, alternative = "less", exact = FALSE)$p.value <= 0.01

    # Check results for one-sided test
    if (p1 == TRUE & p2 == FALSE) {
      a[i,3] = paste("\\textbf{",a[i,3],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      a[i,3] = paste("\\textit{",a[i,3],"}", sep = "")
    } else {
    }

    # effect size
    if (postgres_avmd == "large") {
      a[i,3] = paste("$^{\\ast\\ast\\ast}$",a[i,3], sep = "")
    } else if (postgres_avmd == "medium") {
      a[i,3] = paste("$^{\\ast\\ast}$",a[i,3], sep = "")
    } else if (postgres_avmd == "small") {
      a[i,3] = paste("$^{\\ast}$",a[i,3], sep = "")
    } else {

    }

    # U-test Random vs DR
    rand_coverage <- (rand %>% dplyr::filter(dbms == "Postgres"))$coverage

    p1 <- wilcox.test(dr_coverage, rand_coverage, alternative = "greater", exact = FALSE)$p.value <= 0.01
    p2 <- wilcox.test(dr_coverage, rand_coverage, alternative = "less", exact = FALSE)$p.value <= 0.01

    # check one-sided
    if (p1 == TRUE & p2 == FALSE) {
      a[i,4] = paste("\\textbf{",a[i,4],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      a[i,4] = paste("\\textit{",a[i,4],"}", sep = "")
    } else {
    }
    # effect size
    if (postgres_rand == "large") {
      a[i,4] = paste("$^{\\ast\\ast\\ast}$",a[i,4], sep = "")
    } else if (postgres_rand == "medium") {
      a[i,4] = paste("$^{\\ast\\ast}$",a[i,4], sep = "")
    } else if (postgres_rand == "small") {
      a[i,4] = paste("$^{\\ast}$",a[i,4], sep = "")
    } else {
    }

    # get SQLite effect size
    sqlite_avm <- directedRandomR::effectsize_accurate((dr %>% dplyr::filter(dbms == "SQLite"))$coverage, (avm %>% dplyr::filter(dbms == "SQLite"))$coverage)$size
    sqlite_avmd <- directedRandomR::effectsize_accurate((dr %>% dplyr::filter(dbms == "SQLite"))$coverage, (avmd %>% dplyr::filter(dbms == "SQLite"))$coverage)$size
    sqlite_rand <- directedRandomR::effectsize_accurate((dr %>% dplyr::filter(dbms == "SQLite"))$coverage, (rand %>% dplyr::filter(dbms == "SQLite"))$coverage)$size

    # U-test AVMR vs DR
    dr_coverage <- (dr %>% dplyr::filter(dbms == "SQLite"))$coverage
    avmr_coverage <- (avm %>% dplyr::filter(dbms == "SQLite"))$coverage

    p1 <- wilcox.test(dr_coverage, avmr_coverage, alternative = "greater", exact = FALSE)$p.value <= 0.01
    p2 <- wilcox.test(dr_coverage, avmr_coverage, alternative = "less", exact = FALSE)$p.value <= 0.01

    if (p1 == TRUE & p2 == FALSE) {
      b[i,2] = paste("\\textbf{",b[i,2],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      b[i,2] = paste("\\textit{",b[i,2],"}", sep = "")
    } else {
    }

    if (sqlite_avm == "large") {
      b[i,2] = paste("$^{\\ast\\ast\\ast}$",b[i,2], sep = "")
    } else if (sqlite_avm == "medium") {
      b[i,2] = paste("$^{\\ast\\ast}$",b[i,2], sep = "")
    } else if (sqlite_avm == "small") {
      b[i,2] = paste("$^{\\ast}$",b[i,2], sep = "")
    } else {

    }

    # U-test AVMD vs DR
    avmd_coverage <- (avmd %>% dplyr::filter(dbms == "SQLite"))$coverage

    p1 <- wilcox.test(dr_coverage, avmd_coverage, alternative = "greater", exact = FALSE)$p.value <= 0.01
    p2 <- wilcox.test(dr_coverage, avmd_coverage, alternative = "less", exact = FALSE)$p.value <= 0.01

    if (p1 == TRUE & p2 == FALSE) {
      b[i,3] = paste("\\textbf{",b[i,3],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      b[i,3] = paste("\\textit{",b[i,3],"}", sep = "")
    } else {
    }

    if (sqlite_avmd == "large") {
      b[i,3] = paste("$^{\\ast\\ast\\ast}$",b[i,3], sep = "")
    } else if (sqlite_avmd == "medium") {
      b[i,3] = paste("$^{\\ast\\ast}$",b[i,3], sep = "")
    } else if (sqlite_avmd == "small") {
      b[i,3] = paste("$^{\\ast}$",b[i,3], sep = "")
    } else {

    }

    # U-test Random vs DR
    rand_coverage <- (rand %>% dplyr::filter(dbms == "SQLite"))$coverage

    p1 <- wilcox.test(dr_coverage, rand_coverage, alternative = "greater", exact = FALSE)$p.value <= 0.01
    p2 <- wilcox.test(dr_coverage, rand_coverage, alternative = "less", exact = FALSE)$p.value <= 0.01

    if (p1 == TRUE & p2 == FALSE) {
      b[i,4] = paste("\\textbf{",b[i,4],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      b[i,4] = paste("\\textit{",b[i,4],"}", sep = "")
    } else {
    }

    if (sqlite_rand == "large") {
      b[i,4] = paste("$^{\\ast\\ast\\ast}$",b[i,4], sep = "")
    } else if (sqlite_rand == "medium") {
      b[i,4] = paste("$^{\\ast\\ast}$",b[i,4], sep = "")
    } else if (sqlite_rand == "small") {
      b[i,4] = paste("$^{\\ast}$",b[i,4], sep = "")
    } else {

    }
    # calculate effect size for coverage for HSQL
    hsql_avm <- directedRandomR::effectsize_accurate((dr %>% dplyr::filter(dbms == "HyperSQL"))$coverage, (avm %>% dplyr::filter(dbms == "HyperSQL"))$coverage)$size
    hsql_avmd <- directedRandomR::effectsize_accurate((dr %>% dplyr::filter(dbms == "HyperSQL"))$coverage, (avmd %>% dplyr::filter(dbms == "HyperSQL"))$coverage)$size
    hsql_rand <- directedRandomR::effectsize_accurate((dr %>% dplyr::filter(dbms == "HyperSQL"))$coverage, (rand %>% dplyr::filter(dbms == "HyperSQL"))$coverage)$size

    # U-test AVMR vs DR
    dr_coverage <- (dr %>% dplyr::filter(dbms == "HyperSQL"))$coverage
    avmr_coverage <- (avm %>% dplyr::filter(dbms == "HyperSQL"))$coverage

    p1 <- wilcox.test(dr_coverage, avmr_coverage, alternative = "greater", exact = FALSE)$p.value <= 0.01
    p2 <- wilcox.test(dr_coverage, avmr_coverage, alternative = "less", exact = FALSE)$p.value <= 0.01

    if (p1 == TRUE & p2 == FALSE) {
      c[i,2] = paste("\\textbf{",c[i,2],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      c[i,2] = paste("\\textit{",c[i,2],"}", sep = "")
    } else {
    }

    # effect size
    if (hsql_avm == "large") {
      c[i,2] = paste("$^{\\ast\\ast\\ast}$",c[i,2], sep = "")
    } else if (hsql_avm == "medium") {
      c[i,2] = paste("$^{\\ast\\ast}$",c[i,2], sep = "")
    } else if (hsql_avm == "small") {
      c[i,2] = paste("$^{\\ast}$",c[i,2], sep = "")
    } else {

    }
    avmd_coverage <- (avmd %>% dplyr::filter(dbms == "HyperSQL"))$coverage

    p1 <- wilcox.test(dr_coverage, avmd_coverage, alternative = "greater", exact = FALSE)$p.value <= 0.01
    p2 <- wilcox.test(dr_coverage, avmd_coverage, alternative = "less", exact = FALSE)$p.value <= 0.01

    if (p1 == TRUE & p2 == FALSE) {
      c[i,3] = paste("\\textbf{",c[i,3],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      c[i,3] = paste("\\textit{",c[i,3],"}", sep = "")
    } else {
    }

    if (hsql_avmd == "large") {
      c[i,3] = paste("$^{\\ast\\ast\\ast}$",c[i,3], sep = "")
    } else if (hsql_avmd == "medium") {
      c[i,3] = paste("$^{\\ast\\ast}$",c[i,3], sep = "")
    } else if (hsql_avmd == "small") {
      c[i,3] = paste("$^{\\ast}$",c[i,3], sep = "")
    } else {

    }

    # U-test Random vs DR
    rand_coverage <- (rand %>% dplyr::filter(dbms == "HyperSQL"))$coverage

    p1 <- wilcox.test(dr_coverage, rand_coverage, alternative = "greater", exact = FALSE)$p.value <= 0.01
    p2 <- wilcox.test(dr_coverage, rand_coverage, alternative = "less", exact = FALSE)$p.value <= 0.01

    if (p1 == TRUE & p2 == FALSE) {
      c[i,4] = paste("\\textbf{",c[i,4],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      c[i,4] = paste("\\textit{",c[i,4],"}", sep = "")
    } else {
    }

    if (hsql_rand == "large") {
      c[i,4] = paste("$^{\\ast\\ast\\ast}$",c[i,4], sep = "")
    } else if (hsql_rand == "medium") {
      c[i,4] = paste("$^{\\ast\\ast}$",c[i,4], sep = "")
    } else if (hsql_rand == "small") {
      c[i,4] = paste("$^{\\ast}$",c[i,4], sep = "")
    } else {

    }

    # for latex purposes
    if (a1[i,] == "NistXTS749") {
      a1[i,] <- "NistXTSNine"
    }
    a1[i,] <- paste("\\", a1[i,], "ForTable", sep = "")
  }
  # Combain data
  a <- a[c(1,4,3,2)]
  b <- b[c(1,4,3,2)]
  c <- c[c(1,4,3,2)]
  # With HSQL
  #d <- cbind(a1,c,a,b)
  # Without HSQL
  d <- cbind(a1,a,b)
  if (rtrn == "tex") {
    return(print(xtable::xtable(d), include.rownames=FALSE ,sanitize.text.function = function(x){x}))
  } else {
    return(d)
  }
}

#' FUNCTION: table_generator_timing
#'
#' Generates a latex table or data frame for test generation timing table with effect size and U test.
#' @param d Data frame of analysis
#' @param rtrn Latex (tex) or a data frame (data)
#' @param m Results shown as median or mean
#' @return A A12 effect size and U-test of test generation timing compared pair wise
#' @importFrom magrittr %>%
#' @export
table_generator_timing <- function(d, rtrn = "tex", m = "median") {
  # Arrange dataframe by case study
  d <- d %>% dplyr::arrange(casestudy)
  # copy values for Sig without transforming
  d3 <- d
  # Transform data with rounding down
  d1 <- directedRandomR::transform_execution_times_for_threshold(d, 1000)
  # generate a DF for mean or median
  if (m == "mean") {
    d <- d %>% dplyr::select(dbms, casestudy, datagenerator, testgenerationtime, randomseed) %>% dplyr::group_by(dbms, casestudy, datagenerator) %>% dplyr::summarise(testgenerationtime = format(round((mean(testgenerationtime) / 1000), 2), nsmall = 2))
  } else {
    d <- d %>% dplyr::select(dbms, casestudy, datagenerator, testgenerationtime, randomseed) %>% dplyr::group_by(dbms, casestudy, datagenerator) %>% dplyr::summarise(testgenerationtime = format(round((median(testgenerationtime) / 1000), 2), nsmall = 2))
  }
  # filp the data frame
  d <- reshape2::dcast(d, casestudy ~ dbms + datagenerator, value.var=c("testgenerationtime"))
  # get header
  a1 <- d[1]
  # Split by DBMS
  d2 <- d[2:13]
  d <- d2[ , order(names(d2))]
  c <- d[1:4]
  c <- c[c(3,1,2,4)]
  a <- d[5:8]
  a <- a[c(3,1,2,4)]
  b <- d[9:12]
  b <- b[c(3,1,2,4)]
  # get nunber of rows and itrate through them
  numberOfRows <- nrow(d)
  # change the schemas from fectors to char
  a1$casestudy <- as.character(a1$casestudy)
  for (i in 1:numberOfRows) {
    schema <- a1[i,]
    # get each generators for transformed data
    dr <- d1 %>% dplyr::filter(casestudy == schema, datagenerator == "directedRandom")
    avm <- d1 %>% dplyr::filter(casestudy == schema, datagenerator == "avs")
    avmd <- d1 %>% dplyr::filter(casestudy == schema, datagenerator == "avsDefaults")
    rand <- d1 %>% dplyr::filter(casestudy == schema, datagenerator == "random")

    # Effect size for PSQL
    postgres_avm <- directedRandomR::effectsize_accurate((dr %>% dplyr::filter(dbms == "Postgres"))$testgenerationtime, (avm %>% dplyr::filter(dbms == "Postgres"))$testgenerationtime)$size
    postgres_avmd <- directedRandomR::effectsize_accurate((dr %>% dplyr::filter(dbms == "Postgres"))$testgenerationtime, (avmd %>% dplyr::filter(dbms == "Postgres"))$testgenerationtime)$size
    postgres_rand <- directedRandomR::effectsize_accurate((dr %>% dplyr::filter(dbms == "Postgres"))$testgenerationtime, (rand %>% dplyr::filter(dbms == "Postgres"))$testgenerationtime)$size

    # get generators for non-transformed
    drp <- d3 %>% dplyr::filter(casestudy == schema, datagenerator == "directedRandom")
    avmp <- d3 %>% dplyr::filter(casestudy == schema, datagenerator == "avs")
    avmdp <- d3 %>% dplyr::filter(casestudy == schema, datagenerator == "avsDefaults")
    randp <- d3 %>% dplyr::filter(casestudy == schema, datagenerator == "random")

    dr_time <- (drp %>% dplyr::filter(dbms == "Postgres"))$testgenerationtime
    avmr_time <- (avmp %>% dplyr::filter(dbms == "Postgres"))$testgenerationtime
    # U-test AVMR vs DR
    p1 <- wilcox.test(dr_time, avmr_time, alternative = "greater", exact = FALSE)$p.value >= 0.01
    p2 <- wilcox.test(dr_time, avmr_time, alternative = "less", exact = FALSE)$p.value >= 0.01

    # Check one-sided test
    if (p1 == TRUE & p2 == FALSE) {
      a[i,2] = paste("\\textbf{",a[i,2],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      a[i,2] = paste("\\textit{",a[i,2],"}", sep = "")
    } else {
    }

    if (postgres_avm == "large") {
      a[i,2] = paste("$^{\\ast\\ast\\ast}$",a[i,2], sep = "")
    } else if (postgres_avm == "medium") {
      a[i,2] = paste("$^{\\ast\\ast}$",a[i,2], sep = "")
    } else if (postgres_avm == "small") {
      a[i,2] = paste("$^{\\ast}$",a[i,2], sep = "")
    } else {

    }

    # AVM-D
    avmd_time <- (avmdp %>% dplyr::filter(dbms == "Postgres"))$testgenerationtime

    p1 <- wilcox.test(dr_time, avmd_time, alternative = "greater", exact = FALSE)$p.value >= 0.01
    p2 <- wilcox.test(dr_time, avmd_time, alternative = "less", exact = FALSE)$p.value >= 0.01

    # Check one-sided test
    if (p1 == TRUE & p2 == FALSE) {
      a[i,3] = paste("\\textbf{",a[i,3],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      a[i,3] = paste("\\textit{",a[i,3],"}", sep = "")
    } else {
    }

    if (postgres_avmd == "large") {
      a[i,3] = paste("$^{\\ast\\ast\\ast}$",a[i,3], sep = "")
    } else if (postgres_avmd == "medium") {
      a[i,3] = paste("$^{\\ast\\ast}$",a[i,3], sep = "")
    } else if (postgres_avmd == "small") {
      a[i,3] = paste("$^{\\ast}$",a[i,3], sep = "")
    } else {

    }

    # RANDOM
    rand_time <- (randp %>% dplyr::filter(dbms == "Postgres"))$testgenerationtime

    p1 <- wilcox.test(dr_time, rand_time, alternative = "greater", exact = FALSE)$p.value >= 0.01
    p2 <- wilcox.test(dr_time, rand_time, alternative = "less", exact = FALSE)$p.value >= 0.01

    # Check one-sided test
    if (p1 == TRUE & p2 == FALSE) {
      a[i,4] = paste("\\textbf{",a[i,4],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      a[i,4] = paste("\\textit{",a[i,4],"}", sep = "")
    } else {
    }

    if (postgres_rand == "large") {
      a[i,4] = paste("$^{\\ast\\ast\\ast}$",a[i,4], sep = "")
    } else if (postgres_rand == "medium") {
      a[i,4] = paste("$^{\\ast\\ast}$",a[i,4], sep = "")
    } else if (postgres_rand == "small") {
      a[i,4] = paste("$^{\\ast}$",a[i,4], sep = "")
    } else {
    }

    # Effect size for SQLite
    sqlite_avm <- directedRandomR::effectsize_accurate((dr %>% dplyr::filter(dbms == "SQLite"))$testgenerationtime, (avm %>% dplyr::filter(dbms == "SQLite"))$testgenerationtime)$size
    sqlite_avmd <- directedRandomR::effectsize_accurate((dr %>% dplyr::filter(dbms == "SQLite"))$testgenerationtime, (avmd %>% dplyr::filter(dbms == "SQLite"))$testgenerationtime)$size
    sqlite_rand <- directedRandomR::effectsize_accurate((dr %>% dplyr::filter(dbms == "SQLite"))$testgenerationtime, (rand %>% dplyr::filter(dbms == "SQLite"))$testgenerationtime)$size

    # DR vs AVM-R U-test
    dr_time <- (drp %>% dplyr::filter(dbms == "SQLite"))$testgenerationtime
    avmr_time <- (avmp %>% dplyr::filter(dbms == "SQLite"))$testgenerationtime

    p1 <- wilcox.test(dr_time, avmr_time, alternative = "greater", exact = FALSE)$p.value >= 0.01
    p2 <- wilcox.test(dr_time, avmr_time, alternative = "less", exact = FALSE)$p.value >= 0.01

    # Check one-sided test
    if (p1 == TRUE & p2 == FALSE) {
      b[i,2] = paste("\\textbf{",b[i,2],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      b[i,2] = paste("\\textit{",b[i,2],"}", sep = "")
    } else {
    }

    if (sqlite_avm == "large") {
      b[i,2] = paste("$^{\\ast\\ast\\ast}$",b[i,2], sep = "")
    } else if (sqlite_avm == "medium") {
      b[i,2] = paste("$^{\\ast\\ast}$",b[i,2], sep = "")
    } else if (sqlite_avm == "small") {
      b[i,2] = paste("$^{\\ast}$",b[i,2], sep = "")
    } else {

    }

    # AVM-D u-test
    avmd_time <- (avmdp %>% dplyr::filter(dbms == "SQLite"))$testgenerationtime

    p1 <- wilcox.test(dr_time, avmd_time, alternative = "greater", exact = FALSE)$p.value >= 0.01
    p2 <- wilcox.test(dr_time, avmd_time, alternative = "less", exact = FALSE)$p.value >= 0.01


    # Check one-sided test
    if (p1 == TRUE & p2 == FALSE) {
      b[i,3] = paste("\\textbf{",b[i,3],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      b[i,3] = paste("\\textit{",b[i,3],"}", sep = "")
    } else {
    }

    if (sqlite_avmd == "large") {
      b[i,3] = paste("$^{\\ast\\ast\\ast}$",b[i,3], sep = "")
    } else if (sqlite_avmd == "medium") {
      b[i,3] = paste("$^{\\ast\\ast}$",b[i,3], sep = "")
    } else if (sqlite_avmd == "small") {
      b[i,3] = paste("$^{\\ast}$",b[i,3], sep = "")
    } else {

    }

    # Random U-Test
    rand_time <- (randp %>% dplyr::filter(dbms == "SQLite"))$testgenerationtime

    p1 <- wilcox.test(dr_time, avmd_time, alternative = "greater", exact = FALSE)$p.value >= 0.01
    p2 <- wilcox.test(dr_time, avmd_time, alternative = "less", exact = FALSE)$p.value >= 0.01

    if (p1 == TRUE & p2 == FALSE) {
      b[i,4] = paste("\\textbf{",b[i,4],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      b[i,4] = paste("\\textit{",b[i,4],"}", sep = "")
    } else {
    }

    if (sqlite_rand == "large") {
      b[i,4] = paste("$^{\\ast\\ast\\ast}$",b[i,4], sep = "")
    } else if (sqlite_rand == "medium") {
      b[i,4] = paste("$^{\\ast\\ast}$",b[i,4], sep = "")
    } else if (sqlite_rand == "small") {
      b[i,4] = paste("$^{\\ast}$",b[i,4], sep = "")
    } else {

    }

    # Effect size for HSQL
    hsql_avm <- directedRandomR::effectsize_accurate((dr %>% dplyr::filter(dbms == "HyperSQL"))$testgenerationtime, (avm %>% dplyr::filter(dbms == "HyperSQL"))$testgenerationtime)$size
    hsql_avmd <- directedRandomR::effectsize_accurate((dr %>% dplyr::filter(dbms == "HyperSQL"))$testgenerationtime, (avmd %>% dplyr::filter(dbms == "HyperSQL"))$testgenerationtime)$size
    hsql_rand <- directedRandomR::effectsize_accurate((dr %>% dplyr::filter(dbms == "HyperSQL"))$testgenerationtime, (rand %>% dplyr::filter(dbms == "HyperSQL"))$testgenerationtime)$size

    # U-Test avm-r vs dr
    dr_time <- (drp %>% dplyr::filter(dbms == "HyperSQL"))$testgenerationtime
    avmr_time <- (avmp %>% dplyr::filter(dbms == "HyperSQL"))$testgenerationtime

    p1 <- wilcox.test(dr_time, avmr_time, alternative = "greater", exact = FALSE)$p.value >= 0.01
    p2 <- wilcox.test(dr_time, avmr_time, alternative = "less", exact = FALSE)$p.value >= 0.01

    if (p1 == TRUE & p2 == FALSE) {
      c[i,2] = paste("\\textbf{",c[i,2],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      c[i,2] = paste("\\textit{",c[i,2],"}", sep = "")
    } else {
    }

    if (hsql_avm == "large") {
      c[i,2] = paste("$^{\\ast\\ast\\ast}$",c[i,2], sep = "")
    } else if (hsql_avm == "medium") {
      c[i,2] = paste("$^{\\ast\\ast}$",c[i,2], sep = "")
    } else if (hsql_avm == "small") {
      c[i,2] = paste("$^{\\ast}$",c[i,2], sep = "")
    } else {
    }

    # U-Test avm-d
    avmd_time <- (avmdp %>% dplyr::filter(dbms == "HyperSQL"))$testgenerationtime

    p1 <- wilcox.test(dr_time, avmd_time, alternative = "greater", exact = FALSE)$p.value >= 0.01
    p2 <- wilcox.test(dr_time, avmd_time, alternative = "less", exact = FALSE)$p.value >= 0.01

    if (p1 == TRUE & p2 == FALSE) {
      c[i,3] = paste("\\textbf{",c[i,3],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      c[i,3] = paste("\\textit{",c[i,3],"}", sep = "")
    } else {
    }

    if (hsql_avmd == "large") {
      c[i,3] = paste("$^{\\ast\\ast\\ast}$",c[i,3], sep = "")
    } else if (hsql_avmd == "medium") {
      c[i,3] = paste("$^{\\ast\\ast}$",c[i,3], sep = "")
    } else if (hsql_avmd == "small") {
      c[i,3] = paste("$^{\\ast}$",c[i,3], sep = "")
    } else {

    }

    # Rnadom u-test
    rand_time <- (randp %>% dplyr::filter(dbms == "HyperSQL"))$testgenerationtime

    p1 <- wilcox.test(dr_time, rand_time, alternative = "greater", exact = FALSE)$p.value >= 0.01
    p2 <- wilcox.test(dr_time, rand_time, alternative = "less", exact = FALSE)$p.value >= 0.01

    if (p1 == TRUE & p2 == FALSE) {
      c[i,4] = paste("\\textbf{",c[i,4],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      c[i,4] = paste("\\textit{",c[i,4],"}", sep = "")
    } else {
    }

    if (hsql_rand == "large") {
      c[i,4] = paste("$^{\\ast\\ast\\ast}$",c[i,4], sep = "")
    } else if (hsql_rand == "medium") {
      c[i,4] = paste("$^{\\ast\\ast}$",c[i,4], sep = "")
    } else if (hsql_rand == "small") {
      c[i,4] = paste("$^{\\ast}$",c[i,4], sep = "")
    } else {

    }

    # for latex purposes
    if (a1[i,] == "NistXTS749") {
      a1[i,] <- "NistXTSNine"
    }
    a1[i,] <- paste("\\", a1[i,], "ForTable", sep = "")
  }
  # Combain data
  a <- a[c(1,4,3,2)]
  b <- b[c(1,4,3,2)]
  c <- c[c(1,4,3,2)]

  # With HSQL
  #d <- cbind(a1,c,a,b)
  # Without HSQL
  d <- cbind(a1,a,b)
  #return(d)
  if (rtrn == "tex") {
    return(print(xtable::xtable(d), include.rownames=FALSE ,sanitize.text.function = function(x){x}))
  } else {
    return(d)
  }
}

#' FUNCTION: table_generator_mutation_score
#'
#' Generates a latex table or data frame for mutation score per schema table with effect size and U test.
#' @param d Data frame of mutants
#' @param rtrn Latex (tex) or a data frame (data)
#' @param m Results shown as median or mean
#' @return A A12 effect size and U-test of mutation score per schema compared pair wise
#' @importFrom magrittr %>%
#' @export
table_generator_mutation_score <- function(d, rtrn = "tex", m = "median") {
  # ordering mutants per run
  d <- ordering_mutants_per_schema(d)
  # copying data frame so it can be compared for A12 and U-test
  d1 <- d
  if (m == "mean") {
    d <- d %>% dplyr::group_by(schema, generator, dbms)  %>% dplyr::summarise(mutationScore = format(round(mean(mutationScore), 1), nsmall = 1))
  } else {
    d <- d %>% dplyr::group_by(schema, generator, dbms)  %>% dplyr::summarise(mutationScore = format(round(median(mutationScore), 1), nsmall = 1))
  }
  # Reshaping data frame
  d <- reshape2::dcast(d, schema ~ dbms + generator, value.var=c("mutationScore"))
  a1 <- d[1]
  # Splitting data frame per DBMS
  d2 <- d[2:13]
  d <- d2[ , order(names(d2))]
  c <- d[1:4]
  c <- c[c(3,1,2,4)]
  a <- d[5:8]
  a <- a[c(3,1,2,4)]
  b <- d[9:12]
  b <- b[c(3,1,2,4)]
  # Schemas changed to
  a1$schema <- as.character(a1$schema)
  numberOfRows <- nrow(d)
  for (i in 1:numberOfRows) {
    schema1 <- a1[i,]
    dr <- d1 %>% dplyr::filter(schema == schema1, generator == "directedRandom")
    avm <- d1 %>% dplyr::filter(schema == schema1, generator == "avs")
    avmd <- d1 %>% dplyr::filter(schema == schema1, generator == "avsDefaults")
    rand <- d1 %>% dplyr::filter(schema == schema1, generator == "random")

    # PSQL A12
    postgres_avm <- directedRandomR::effectsize_accurate((dr %>% dplyr::filter(dbms == "Postgres"))$mutationScore, (avm %>% dplyr::filter(dbms == "Postgres"))$mutationScore)$size
    postgres_avmd <- directedRandomR::effectsize_accurate((dr %>% dplyr::filter(dbms == "Postgres"))$mutationScore, (avmd %>% dplyr::filter(dbms == "Postgres"))$mutationScore)$size
    postgres_rand <- directedRandomR::effectsize_accurate((dr %>% dplyr::filter(dbms == "Postgres"))$mutationScore, (rand %>% dplyr::filter(dbms == "Postgres"))$mutationScore)$size

    # DR vs AVM-r
    dr_mutation <- (dr %>% dplyr::filter(dbms == "Postgres"))$mutationScore
    avmr_mutation <- (avm %>% dplyr::filter(dbms == "Postgres"))$mutationScore

    p1 <- wilcox.test(dr_mutation, avmr_mutation, alternative = "greater", exact = FALSE)$p.value <= 0.01
    p2 <- wilcox.test(dr_mutation, avmr_mutation, alternative = "less", exact = FALSE)$p.value <= 0.01

    if (p1 == TRUE & p2 == FALSE) {
      a[i,2] = paste("\\textbf{",a[i,2],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      a[i,2] = paste("\\textit{",a[i,2],"}", sep = "")
    } else {
    }

    if (postgres_avm == "large") {
      a[i,2] = paste("$^{\\ast\\ast\\ast}$",a[i,2], sep = "")
    } else if (postgres_avm == "medium") {
      a[i,2] = paste("$^{\\ast\\ast}$",a[i,2], sep = "")
    } else if (postgres_avm == "small") {
      a[i,2] = paste("$^{\\ast}$",a[i,2], sep = "")
    } else {
      a[i,2] = paste("$",a[i,2],"$", sep = "")
    }

    # AVM-D vs DR
    avmd_mutation <- (avmd %>% dplyr::filter(dbms == "Postgres"))$mutationScore

    p1 <- wilcox.test(dr_mutation, avmd_mutation, alternative = "greater", exact = FALSE)$p.value <= 0.01
    p2 <- wilcox.test(dr_mutation, avmd_mutation, alternative = "less", exact = FALSE)$p.value <= 0.01

    if (p1 == TRUE & p2 == FALSE) {
      a[i,3] = paste("\\textbf{",a[i,3],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      a[i,3] = paste("\\textit{",a[i,3],"}", sep = "")
    } else {
    }

    if (postgres_avmd == "large") {
      a[i,3] = paste("$^{\\ast\\ast\\ast}$",a[i,3], sep = "")
    } else if (postgres_avmd == "medium") {
      a[i,3] = paste("$^{\\ast\\ast}$",a[i,3], sep = "")
    } else if (postgres_avmd == "small") {
      a[i,3] = paste("$^{\\ast}$",a[i,3], sep = "")
    } else {

    }

    # Random vs DR
    rand_mutation <- (rand %>% dplyr::filter(dbms == "Postgres"))$mutationScore

    p1 <- wilcox.test(dr_mutation, rand_mutation, alternative = "greater", exact = FALSE)$p.value <= 0.01
    p2 <- wilcox.test(dr_mutation, rand_mutation, alternative = "less", exact = FALSE)$p.value <= 0.01

    if (p1 == TRUE & p2 == FALSE) {
      a[i,4] = paste("\\textbf{",a[i,4],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      a[i,4] = paste("\\textit{",a[i,4],"}", sep = "")
    } else {
    }

    if (postgres_rand == "large") {
      a[i,4] = paste("$^{\\ast\\ast\\ast}$",a[i,4], sep = "")
    } else if (postgres_rand == "medium") {
      a[i,4] = paste("$^{\\ast\\ast}$",a[i,4], sep = "")
    } else if (postgres_rand == "small") {
      a[i,4] = paste("$^{\\ast}$",a[i,4], sep = "")
    } else {

    }

    # A12 SQLite
    sqlite_avm <- directedRandomR::effectsize_accurate((dr %>% dplyr::filter(dbms == "SQLite"))$mutationScore, (avm %>% dplyr::filter(dbms == "SQLite"))$mutationScore)$size
    sqlite_avmd <- directedRandomR::effectsize_accurate((dr %>% dplyr::filter(dbms == "SQLite"))$mutationScore, (avmd %>% dplyr::filter(dbms == "SQLite"))$mutationScore)$size
    sqlite_rand <- directedRandomR::effectsize_accurate((dr %>% dplyr::filter(dbms == "SQLite"))$mutationScore, (rand %>% dplyr::filter(dbms == "SQLite"))$mutationScore)$size

    # Dr vs AVM-R
    dr_mutation <- (dr %>% dplyr::filter(dbms == "SQLite"))$mutationScore
    avmr_mutation <- (avm %>% dplyr::filter(dbms == "SQLite"))$mutationScore

    p1 <- wilcox.test(dr_mutation, avmr_mutation, alternative = "greater", exact = FALSE)$p.value <= 0.01
    p2 <- wilcox.test(dr_mutation, avmr_mutation, alternative = "less", exact = FALSE)$p.value <= 0.01

    if (p1 == TRUE & p2 == FALSE) {
      b[i,2] = paste("\\textbf{",b[i,2],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      b[i,2] = paste("\\textit{",b[i,2],"}", sep = "")
    } else {
    }

    if (sqlite_avm == "large") {
      b[i,2] = paste("$^{\\ast\\ast\\ast}$",b[i,2], sep = "")
    } else if (sqlite_avm == "medium") {
      b[i,2] = paste("$^{\\ast\\ast}$",b[i,2], sep = "")
    } else if (sqlite_avm == "small") {
      b[i,2] = paste("$^{\\ast}$",b[i,2], sep = "")
    } else {

    }

    # AVMD vs DR
    avmd_mutation <- (avmd %>% dplyr::filter(dbms == "SQLite"))$mutationScore

    p1 <- wilcox.test(dr_mutation, avmd_mutation, alternative = "greater", exact = FALSE)$p.value <= 0.01
    p2 <- wilcox.test(dr_mutation, avmd_mutation, alternative = "less", exact = FALSE)$p.value <= 0.01

    if (p1 == TRUE & p2 == FALSE) {
      b[i,3] = paste("\\textbf{",b[i,3],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      b[i,3] = paste("\\textit{",b[i,3],"}", sep = "")
    } else {
    }

    if (sqlite_avmd == "large") {
      b[i,3] = paste("$^{\\ast\\ast\\ast}$",b[i,3], sep = "")
    } else if (sqlite_avmd == "medium") {
      b[i,3] = paste("$^{\\ast\\ast}$",b[i,3], sep = "")
    } else if (sqlite_avmd == "small") {
      b[i,3] = paste("$^{\\ast}$",b[i,3], sep = "")
    } else {

    }

    # Random vs DR
    rand_mutation <- (rand %>% dplyr::filter(dbms == "SQLite"))$mutationScore

    p1 <- wilcox.test(dr_mutation, rand_mutation, alternative = "greater", exact = FALSE)$p.value <= 0.01
    p2 <- wilcox.test(dr_mutation, rand_mutation, alternative = "less", exact = FALSE)$p.value <= 0.01

    if (p1 == TRUE & p2 == FALSE) {
      b[i,4] = paste("\\textbf{",b[i,4],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      b[i,4] = paste("\\textit{",b[i,4],"}", sep = "")
    } else {
    }

    if (sqlite_rand == "large") {
      b[i,4] = paste("$^{\\ast\\ast\\ast}$",b[i,4], sep = "")
    } else if (sqlite_rand == "medium") {
      b[i,4] = paste("$^{\\ast\\ast}$",b[i,4], sep = "")
    } else if (sqlite_rand == "small") {
      b[i,4] = paste("$^{\\ast}$",b[i,4], sep = "")
    } else {

    }

    # Effect size for HSQL
    hsql_avm <- directedRandomR::effectsize_accurate((dr %>% dplyr::filter(dbms == "HyperSQL"))$mutationScore, (avm %>% dplyr::filter(dbms == "HyperSQL"))$mutationScore)$size
    hsql_avmd <- directedRandomR::effectsize_accurate((dr %>% dplyr::filter(dbms == "HyperSQL"))$mutationScore, (avmd %>% dplyr::filter(dbms == "HyperSQL"))$mutationScore)$size
    hsql_rand <- directedRandomR::effectsize_accurate((dr %>% dplyr::filter(dbms == "HyperSQL"))$mutationScore, (rand %>% dplyr::filter(dbms == "HyperSQL"))$mutationScore)$size

    # DR vs AVMR
    dr_mutation <- (dr %>% dplyr::filter(dbms == "HyperSQL"))$mutationScore
    avmr_mutation <- (avm %>% dplyr::filter(dbms == "HyperSQL"))$mutationScore

    p1 <- wilcox.test(dr_mutation, avmr_mutation, alternative = "greater", exact = FALSE)$p.value <= 0.01
    p2 <- wilcox.test(dr_mutation, avmr_mutation, alternative = "less", exact = FALSE)$p.value <= 0.01

    if (p1 == TRUE & p2 == FALSE) {
      c[i,2] = paste("\\textbf{",c[i,2],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      c[i,2] = paste("\\textit{",c[i,2],"}", sep = "")
    } else {
    }

    if (hsql_avm == "large") {
      c[i,2] = paste("$^{\\ast\\ast\\ast}$",c[i,2], sep = "")
    } else if (hsql_avm == "medium") {
      c[i,2] = paste("$^{\\ast\\ast}$",c[i,2], sep = "")
    } else if (hsql_avm == "small") {
      c[i,2] = paste("$^{\\ast}$",c[i,2], sep = "")
    } else {

    }

    # AVMD vs DR
    avmd_mutation <- (avmd %>% dplyr::filter(dbms == "HyperSQL"))$mutationScore

    p1 <- wilcox.test(dr_mutation, avmd_mutation, alternative = "greater", exact = FALSE)$p.value <= 0.01
    p2 <- wilcox.test(dr_mutation, avmd_mutation, alternative = "less", exact = FALSE)$p.value <= 0.01

    if (p1 == TRUE & p2 == FALSE) {
      c[i,3] = paste("\\textbf{",c[i,3],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      c[i,3] = paste("\\textit{",c[i,3],"}", sep = "")
    } else {
    }

    if (hsql_avmd == "large") {
      c[i,3] = paste("$^{\\ast\\ast\\ast}$",c[i,3], sep = "")
    } else if (hsql_avmd == "medium") {
      c[i,3] = paste("$^{\\ast\\ast}$",c[i,3], sep = "")
    } else if (hsql_avmd == "small") {
      c[i,3] = paste("$^{\\ast}$",c[i,3], sep = "")
    } else {

    }

    # Random vs DR
    rand_mutation <- (rand %>% dplyr::filter(dbms == "HyperSQL"))$mutationScore

    p1 <- wilcox.test(dr_mutation, rand_mutation, alternative = "greater", exact = FALSE)$p.value <= 0.01
    p2 <- wilcox.test(dr_mutation, rand_mutation, alternative = "less", exact = FALSE)$p.value <= 0.01

    if (p1 == TRUE & p2 == FALSE) {
      c[i,4] = paste("\\textbf{",c[i,4],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      c[i,4] = paste("\\textit{",c[i,4],"}", sep = "")
    } else {
    }

    if (hsql_rand == "large") {
      c[i,4] = paste("$^{\\ast\\ast\\ast}$",c[i,4], sep = "")
    } else if (hsql_rand == "medium") {
      c[i,4] = paste("$^{\\ast\\ast}$",c[i,4], sep = "")
    } else if (hsql_rand == "small") {
      c[i,4] = paste("$^{\\ast}$",c[i,4], sep = "")
    } else {

    }

    if (a1[i,] == "NistXTS749") {
      a1[i,] <- "NistXTSNine"
    }
    a1[i,] <- paste("\\", a1[i,], "ForTable", sep = "")
  }
  a <- a[c(1,4,3,2)]
  b <- b[c(1,4,3,2)]
  c <- c[c(1,4,3,2)]
  # With HSQL
  #d <- cbind(a1,c,a,b)
  # Without HSQL
  d <- cbind(a1,a,b)
  if (rtrn == "tex") {
    return(print(xtable::xtable(d), include.rownames=FALSE ,sanitize.text.function = function(x){x}))
  } else {
    return(d)
  }
}


#' FUNCTION: table_generator_mutant_operators
#'
#' Generates a latex table or data frame for mutation operators table with effect size and U test.
#' @param d Data frame of mutants
#' @param rtrn Latex (tex) or a data frame (data)
#' @param m Results shown as median or mean
#' @return A A12 effect size and U-test of mutation score per operator compared pair wise
#' @importFrom magrittr %>%
#' @export
table_generator_mutant_operators <- function(d, rtrn = "tex", m = "median") {
  # Order mutants per run
  d <- ordering_mutants_per_operator(d)
  # copying data before reshaping
  d1 <- d
  if (m == "mean") {
    a <- d %>% dplyr::group_by(dbms, generator, operator) %>% dplyr::summarise(value = format(mean(mutationScore), nsmall = 1))
  } else {
    a <- d %>% dplyr::group_by(dbms, generator, operator) %>% dplyr::summarise(value = format(median(mutationScore), nsmall = 1))
  }
  d <- reshape2::dcast(a,  operator ~ dbms + generator)
  # Spliting df per DBMS
  a1 <- d[1]
  d2 <- d[2:13]
  d <- d2[ , order(names(d2))]
  c <- d[1:4]
  c <- c[c(3,1,2,4)]
  a <- d[5:8]
  a <- a[c(3,1,2,4)]
  b <- d[9:12]
  b <- b[c(3,1,2,4)]
  # Transoferming operater columns to strings
  a1$operator <- as.character(a1$operator)
  numberOfRows <- nrow(d)
  for (i in 1:numberOfRows) {
    selected_operator <- a1[i,]
    # Selecting data per DBMS and generator
    postgres_dr <- d1 %>% dplyr::filter(operator == selected_operator, dbms == "Postgres", generator == "directedRandom")
    postgres_avm <- d1 %>% dplyr::filter(operator == selected_operator, dbms == "Postgres", generator == "avs")
    postgres_avmd <- d1 %>% dplyr::filter(operator == selected_operator, dbms == "Postgres", generator == "avsDefaults")
    postgres_rand <- d1 %>% dplyr::filter(operator == selected_operator, dbms == "Postgres", generator == "random")
    sqlite_dr <- d1 %>% dplyr::filter(operator == selected_operator, dbms == "SQLite", generator == "directedRandom")
    sqlite_avm <- d1 %>% dplyr::filter(operator == selected_operator, dbms == "SQLite", generator == "avs")
    sqlite_avmd <- d1 %>% dplyr::filter(operator == selected_operator, dbms == "SQLite", generator == "avsDefaults")
    sqlite_rand <- d1 %>% dplyr::filter(operator == selected_operator, dbms == "SQLite", generator == "random")
    hsql_dr <- d1 %>% dplyr::filter(operator == selected_operator, dbms == "HyperSQL", generator == "directedRandom")
    hsql_avm <- d1 %>% dplyr::filter(operator == selected_operator, dbms == "HyperSQL", generator == "avs")
    hsql_avmd <- d1 %>% dplyr::filter(operator == selected_operator, dbms == "HyperSQL", generator == "avsDefaults")
    hsql_rand <- d1 %>% dplyr::filter(operator == selected_operator, dbms == "HyperSQL", generator == "random")

    # A12 for PSQL
    postgres_avm_effectsize <- directedRandomR::effectsize_accurate(postgres_dr$mutationScore, postgres_avm$mutationScore)$size
    postgres_avmd_effectsize <- directedRandomR::effectsize_accurate(postgres_dr$mutationScore, postgres_avmd$mutationScore)$size
    postgres_rand_effectsize <- directedRandomR::effectsize_accurate(postgres_dr$mutationScore, postgres_rand$mutationScore)$size

    # Sig for DR vs AVMR
    dr_mutation <- postgres_dr$mutationScore
    avmr_mutation <- postgres_avm$mutationScore

    p1 <- wilcox.test(dr_mutation, avmr_mutation, alternative = "greater", exact = FALSE)$p.value <= 0.01
    p2 <- wilcox.test(dr_mutation, avmr_mutation, alternative = "less", exact = FALSE)$p.value <= 0.01

    if (p1 == TRUE & p2 == FALSE) {
      a[i,2] = paste("\\textbf{",a[i,2],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      a[i,2] = paste("\\textit{",a[i,2],"}", sep = "")
    } else {
    }

    if (postgres_avm_effectsize == "large") {
      a[i,2] = paste("$^{\\ast\\ast\\ast}$",a[i,2], sep = "")
    } else if (postgres_avm_effectsize == "medium") {
      a[i,2] = paste("$^{\\ast\\ast}$",a[i,2], sep = "")
    } else if (postgres_avm_effectsize == "small") {
      a[i,2] = paste("$^{\\ast}$",a[i,2], sep = "")
    } else {
    }

    # U-test AVMD vs DR
    avmd_mutation <- postgres_avmd$mutationScore

    p1 <- wilcox.test(dr_mutation, avmd_mutation, alternative = "greater", exact = FALSE)$p.value <= 0.01
    p2 <- wilcox.test(dr_mutation, avmd_mutation, alternative = "less", exact = FALSE)$p.value <= 0.01

    if (p1 == TRUE & p2 == FALSE) {
      a[i,3] = paste("\\textbf{",a[i,3],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      a[i,3] = paste("\\textit{",a[i,3],"}", sep = "")
    } else {
    }

    if (postgres_avmd_effectsize == "large") {
      a[i,3] = paste("$^{\\ast\\ast\\ast}$",a[i,3], sep = "")
    } else if (postgres_avmd_effectsize == "medium") {
      a[i,3] = paste("$^{\\ast\\ast}$",a[i,3], sep = "")
    } else if (postgres_avmd_effectsize == "small") {
      a[i,3] = paste("$^{\\ast}$",a[i,3], sep = "")
    } else {

    }

    # U-test for Random vs DR
    rand_mutation <- postgres_rand$mutationScore

    p1 <- wilcox.test(dr_mutation, rand_mutation, alternative = "greater", exact = FALSE)$p.value <= 0.01
    p2 <- wilcox.test(dr_mutation, rand_mutation, alternative = "less", exact = FALSE)$p.value <= 0.01

    if (p1 == TRUE & p2 == FALSE) {
      a[i,4] = paste("\\textbf{",a[i,4],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      a[i,4] = paste("\\textit{",a[i,4],"}", sep = "")
    } else {
    }

    if (postgres_rand_effectsize == "large") {
      a[i,4] = paste("$^{\\ast\\ast\\ast}$",a[i,4], sep = "")
    } else if (postgres_rand_effectsize == "medium") {
      a[i,4] = paste("$^{\\ast\\ast}$",a[i,4], sep = "")
    } else if (postgres_rand_effectsize == "small") {
      a[i,4] = paste("$^{\\ast}$",a[i,4], sep = "")
    } else {

    }

    # A12 for SQLite
    sqlite_avm_effectsize <- directedRandomR::effectsize_accurate(sqlite_dr$mutationScore, sqlite_avm$mutationScore)$size
    sqlite_avmd_effectsize <- directedRandomR::effectsize_accurate(sqlite_dr$mutationScore, sqlite_avmd$mutationScore)$size
    sqlite_rand_effectsize <- directedRandomR::effectsize_accurate(sqlite_dr$mutationScore, sqlite_rand$mutationScore)$size

    # U-test dr vs avm-r
    dr_mutation <- sqlite_dr$mutationScore
    avmr_mutation <- sqlite_avm$mutationScore

    p1 <- wilcox.test(dr_mutation, avmr_mutation, alternative = "greater", exact = FALSE)$p.value <= 0.01
    p2 <- wilcox.test(dr_mutation, avmr_mutation, alternative = "less", exact = FALSE)$p.value <= 0.01

    if (p1 == TRUE & p2 == FALSE) {
      b[i,2] = paste("\\textbf{",b[i,2],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      b[i,2] = paste("\\textit{",b[i,2],"}", sep = "")
    } else {
    }

    if (sqlite_avm_effectsize == "large") {
      b[i,2] = paste("$^{\\ast\\ast\\ast}$",b[i,2], sep = "")
    } else if (sqlite_avm_effectsize == "medium") {
      b[i,2] = paste("$^{\\ast\\ast}$",b[i,2], sep = "")
    } else if (sqlite_avm_effectsize == "small") {
      b[i,2] = paste("$^{\\ast}$",b[i,2], sep = "")
    } else {

    }

    # U-test AVMD vs dr
    avmd_mutation <- sqlite_avmd$mutationScore

    p1 <- wilcox.test(dr_mutation, avmd_mutation, alternative = "greater", exact = FALSE)$p.value <= 0.01
    p2 <- wilcox.test(dr_mutation, avmd_mutation, alternative = "less", exact = FALSE)$p.value <= 0.01

    if (p1 == TRUE & p2 == FALSE) {
      b[i,3] = paste("\\textbf{",b[i,3],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      b[i,3] = paste("\\textit{",b[i,3],"}", sep = "")
    } else {
    }

    if (sqlite_avmd_effectsize == "large") {
      b[i,3] = paste("$^{\\ast\\ast\\ast}$",b[i,3], sep = "")
    } else if (sqlite_avmd_effectsize == "medium") {
      b[i,3] = paste("$^{\\ast\\ast}$",b[i,3], sep = "")
    } else if (sqlite_avmd_effectsize == "small") {
      b[i,3] = paste("$^{\\ast}$",b[i,3], sep = "")
    } else {

    }

    # U-test for Random vs DR
    rand_mutation <- sqlite_rand$mutationScore

    p1 <- wilcox.test(dr_mutation, rand_mutation, alternative = "greater", exact = FALSE)$p.value <= 0.01
    p2 <- wilcox.test(dr_mutation, rand_mutation, alternative = "less", exact = FALSE)$p.value <= 0.01

    if (p1 == TRUE & p2 == FALSE) {
      b[i,4] = paste("\\textbf{",b[i,4],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      b[i,4] = paste("\\textit{",b[i,4],"}", sep = "")
    } else {
    }

    if (sqlite_rand_effectsize == "large") {
      b[i,4] = paste("$^{\\ast\\ast\\ast}$",b[i,4], sep = "")
    } else if (sqlite_rand_effectsize == "medium") {
      b[i,4] = paste("$^{\\ast\\ast}$",b[i,4], sep = "")
    } else if (sqlite_rand_effectsize == "small") {
      b[i,4] = paste("$^{\\ast}$",b[i,4], sep = "")
    } else {

    }

    # A12 for HSQL
    hsql_avm_effectsize <- directedRandomR::effectsize_accurate(hsql_dr$mutationScore, hsql_avm$mutationScore)$size
    hsql_avmd_effectsize <- directedRandomR::effectsize_accurate(hsql_dr$mutationScore, hsql_avmd$mutationScore)$size
    hsql_rand_effectsize <- directedRandomR::effectsize_accurate(hsql_dr$mutationScore, hsql_rand$mutationScore)$size

    # U-test for DR vs AVM-R
    dr_mutation <- hsql_dr$mutationScore
    avmr_mutation <- hsql_avm$mutationScore

    p1 <- wilcox.test(dr_mutation, avmr_mutation, alternative = "greater", exact = FALSE)$p.value <= 0.01
    p2 <- wilcox.test(dr_mutation, avmr_mutation, alternative = "less", exact = FALSE)$p.value <= 0.01

    if (p1 == TRUE & p2 == FALSE) {
      c[i,2] = paste("\\textbf{",c[i,2],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      c[i,2] = paste("\\textit{",c[i,2],"}", sep = "")
    } else {
    }

    if (hsql_avm_effectsize == "large") {
      c[i,2] = paste("$^{\\ast\\ast\\ast}$",c[i,2], sep = "")
    } else if (hsql_avm_effectsize == "medium") {
      c[i,2] = paste("$^{\\ast\\ast}$",c[i,2], sep = "")
    } else if (hsql_avm_effectsize == "small") {
      c[i,2] = paste("$^{\\ast}$",c[i,2], sep = "")
    } else {

    }

    # AVM-D vs DR U-test
    avmd_mutation <- hsql_avmd$mutationScore
    p1 <- wilcox.test(dr_mutation, avmd_mutation, alternative = "greater", exact = FALSE)$p.value <= 0.01
    p2 <- wilcox.test(dr_mutation, avmd_mutation, alternative = "less", exact = FALSE)$p.value <= 0.01

    if (p1 == TRUE & p2 == FALSE) {
      c[i,3] = paste("\\textbf{",c[i,3],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      c[i,3] = paste("\\textit{",c[i,3],"}", sep = "")
    } else {
    }

    if (hsql_avmd_effectsize == "large") {
      c[i,3] = paste("$^{\\ast\\ast\\ast}$",c[i,3], sep = "")
    } else if (hsql_avmd_effectsize == "medium") {
      c[i,3] = paste("$^{\\ast\\ast}$",c[i,3], sep = "")
    } else if (hsql_avmd_effectsize == "small") {
      c[i,3] = paste("$^{\\ast}$",c[i,3], sep = "")
    } else {

    }

    # Random vs Dr U-test
    rand_mutation <- hsql_rand$mutationScore

    p1 <- wilcox.test(dr_mutation, rand_mutation, alternative = "greater", exact = FALSE)$p.value <= 0.01
    p2 <- wilcox.test(dr_mutation, rand_mutation, alternative = "less", exact = FALSE)$p.value <= 0.01

    if (p1 == TRUE & p2 == FALSE) {
      c[i,4] = paste("\\textbf{",c[i,4],"}", sep = "")
    } else if (p1 == FALSE & p2 == TRUE) {
      c[i,4] = paste("\\textit{",c[i,4],"}", sep = "")
    } else {
    }

    if (hsql_rand_effectsize == "large") {
      c[i,4] = paste("$^{\\ast\\ast\\ast}$",c[i,4], sep = "")
    } else if (hsql_rand_effectsize == "medium") {
      c[i,4] = paste("$^{\\ast\\ast}$",c[i,4], sep = "")
    } else if (hsql_rand_effectsize == "small") {
      c[i,4] = paste("$^{\\ast}$",c[i,4], sep = "")
    } else {

    }

    # For latex table
    a1[i,] <- paste("\\", a1[i,], sep = "")


  }
  a <- a[c(1,4,3,2)]
  b <- b[c(1,4,3,2)]
  c <- c[c(1,4,3,2)]

  # With HSQL
  #d <- cbind(a1,c,a,b)
  # Without HSQL
  d <- cbind(a1,a,b)

  if (rtrn == "tex") {
    return(print(xtable::xtable(d), include.rownames=FALSE ,sanitize.text.function = function(x){x}))
  } else {
    return(d)
  }
}

#' FUNCTION: ordering_mutants_per_schema
#'
#' It generates an ordered data frame of mutants (normal type) grouped by each run per schema and its mutation score.
#' @param d Data frame of mutants
#' @return A data frame of ordred mutants and grouped by runs and mutation score per run per schema
#' @importFrom magrittr %>%
#' @export
ordering_mutants_per_schema <- function(d) {

  # Only selecting normal mutants types
  d1 <- d %>% dplyr::filter(type == "NORMAL")
  dt <- NULL

  # Get each case study
  casestudy <- as.vector(dplyr::distinct(d1, schema))[[1]]
  # Get each DBMS
  dbs <- as.vector(dplyr::distinct(d1, dbms))[[1]]
  for (case in casestudy) {
    schema1 <- case
    for (db in dbs) {
      # Filter data
      filtered_data <- d1 %>% dplyr::filter(schema == schema1, dbms == db) %>% dplyr::group_by(identifier, dbms)
      # Select first schema to be grouped
      first_schema <- filtered_data[1,3]
      test <- NULL

      # Get each run for DR
      ids <- filtered_data %>% dplyr::filter(schema== first_schema[[1,1]], generator == "directedRandom") %>% dplyr::select(identifier,dbms,schema) %>% unique
      ids$number=1:nrow(ids)
      filtered_data %>% left_join(ids, by = c("identifier", "dbms", "schema")) %>% dplyr::mutate(number=as.numeric(ifelse(is.na(number),1,number))) %>% ungroup %>% dplyr::mutate(number = cummax(number)) -> test
      dr_minsitrust <- test %>% dplyr::filter(generator == "directedRandom") %>% dplyr::group_by(identifier, dbms, generator, number, schema) %>% dplyr::summarise(killed_mutants = sum(killed == "true"), total_mutants = (sum(killed == "true") + sum(killed == "false")))
      dr <- dr_minsitrust
      dr <- dr %>% dplyr::group_by(number, generator, dbms, schema) %>% dplyr::summarise(killed_mutants = sum(killed_mutants), total_mutants = sum(total_mutants)) %>% dplyr::mutate(mutationScore = round((killed_mutants/total_mutants) * 100, 2))

      # Get each run for AVM-R
      ids <- filtered_data %>% dplyr::filter(schema== first_schema[[1,1]], generator == "avs") %>% dplyr::select(identifier,dbms,schema) %>% unique
      ids$number=1:nrow(ids)
      filtered_data %>% left_join(ids, by = c("identifier", "dbms", "schema"))  %>% dplyr::mutate(number=as.numeric(ifelse(is.na(number),1,number))) %>% ungroup %>% dplyr::mutate(number = cummax(number)) -> test
      avs_minsitrust <- test %>% dplyr::filter(generator == "avs") %>% dplyr::group_by(identifier, dbms, generator, number, schema) %>% dplyr::summarise(killed_mutants = sum(killed == "true"), total_mutants = (sum(killed == "true") + sum(killed == "false")))
      avm <- avs_minsitrust
      avm <- avm %>% dplyr::group_by(number, generator, dbms, schema) %>% dplyr::summarise(killed_mutants = sum(killed_mutants), total_mutants = sum(total_mutants)) %>% dplyr::mutate(mutationScore = round((killed_mutants/total_mutants) * 100, 2))

      # Get each run for AVM-D
      ids <- filtered_data %>% dplyr::filter(schema== first_schema[[1,1]], generator == "avsDefaults") %>% dplyr::select(identifier,dbms,schema) %>% unique
      ids$number=1:nrow(ids)
      filtered_data %>% left_join(ids, by = c("identifier", "dbms", "schema"))  %>% dplyr::mutate(number=as.numeric(ifelse(is.na(number),1,number))) %>% ungroup %>% dplyr::mutate(number = cummax(number)) -> test
      avsd_minsitrust <- test %>% dplyr::filter(generator == "avsDefaults") %>% dplyr::group_by(identifier, dbms, generator, number, schema) %>% dplyr::summarise(killed_mutants = sum(killed == "true"), total_mutants = (sum(killed == "true") + sum(killed == "false")))
      avmd <- avsd_minsitrust
      avmd <- avmd %>% dplyr::group_by(number, generator, dbms, schema) %>% dplyr::summarise(killed_mutants = sum(killed_mutants), total_mutants = sum(total_mutants)) %>% dplyr::mutate(mutationScore = round((killed_mutants/total_mutants) * 100, 2))

      # Get each run for Random
      ids <- filtered_data %>% dplyr::filter(schema== first_schema[[1,1]], generator == "random") %>% dplyr::select(identifier,dbms,schema) %>% unique
      ids$number=1:nrow(ids)
      filtered_data %>% left_join(ids, by = c("identifier", "dbms", "schema"))  %>% dplyr::mutate(number=as.numeric(ifelse(is.na(number),1,number))) %>% ungroup %>% dplyr::mutate(number = cummax(number)) -> test
      ran_minsitrust <- test %>% dplyr::filter(generator == "random") %>% dplyr::group_by(identifier, dbms, generator, number, schema) %>% dplyr::summarise(killed_mutants = sum(killed == "true"), total_mutants = (sum(killed == "true") + sum(killed == "false")))
      rand <- ran_minsitrust
      rand <- rand %>% dplyr::group_by(number, generator, dbms, schema) %>% dplyr::summarise(killed_mutants = sum(killed_mutants), total_mutants = sum(total_mutants)) %>% dplyr::mutate(mutationScore = round((killed_mutants/total_mutants) * 100, 2))

      # save each run per DBMS
      if (db == "Postgres") {
        postgres_dr <- dr
        postgres_avm <- avm
        postgres_avmd <- avmd
        postgres_rand <- rand
      } else if (db == "SQLite") {
        sqlite_dr <- dr
        sqlite_avm <- avm
        sqlite_avmd <- avmd
        sqlite_rand <- rand
      } else if (db == "HyperSQL") {
        hsql_dr <- dr
        hsql_avm <- avm
        hsql_avmd <- avmd
        hsql_rand <- rand
      }

    }

    # Arrange by Runs (numbers)
    postgres_dr <- dplyr::arrange(postgres_dr, number)
    postgres_avm <- dplyr::arrange(postgres_avm, number)
    postgres_avmd <- dplyr::arrange(postgres_avmd, number)
    postgres_rand <- dplyr::arrange(postgres_rand, number)
    sqlite_dr <- dplyr::arrange(sqlite_dr, number)
    sqlite_avm <- dplyr::arrange(sqlite_avm, number)
    sqlite_avmd <- dplyr::arrange(sqlite_avmd, number)
    sqlite_rand <- dplyr::arrange(sqlite_rand, number)
    hsql_dr <- dplyr::arrange(hsql_dr, number)
    hsql_avm <- dplyr::arrange(hsql_avm, number)
    hsql_avmd <- dplyr::arrange(hsql_avmd, number)
    hsql_rand <- dplyr::arrange(hsql_rand, number)

    # bind them together
    postgres <- rbind(postgres_dr, postgres_avm, postgres_avmd, postgres_rand)
    sqlite <- rbind(sqlite_dr, sqlite_avm, sqlite_avmd, sqlite_rand)
    hsql <- rbind(hsql_dr, hsql_avm, hsql_avmd, hsql_rand)

    dt <- rbind(dt, postgres, sqlite, hsql)
  }
  return(dt)
}

#' FUNCTION: ordering_mutants_per_operator
#'
#' It generates an ordered data frame of mutants (normal type) grouped by each run per operator and its mutation score.
#' @param d Data frame of mutants
#' @return A data frame of ordred mutants and grouped by runs and mutation score per run per operator
#' @importFrom magrittr %>%
#' @export
ordering_mutants_per_operator <- function(d) {
  # Only selecting normal mutants types
  d1 <- d %>% dplyr::filter(type == "NORMAL")
  dt <- NULL

  # get DBMSs
  dbs <- as.vector(distinct(d1, dbms))[[1]]
  # Get Operators
  operators <- as.vector(distinct(d1, operator))[[1]]
  for (selected_operator in operators) {
    for (db in dbs) {
      # Filter Data per operator
      filtered_data <- d1 %>% dplyr::filter(operator == selected_operator, schema != "iTrust", dbms == db) %>% dplyr::group_by(identifier, dbms)
      first_schema <- filtered_data[1,3]
      test <- NULL

      # Get each run for DR
      ids <- filtered_data %>% dplyr::filter(schema== first_schema[[1,1]], generator == "directedRandom") %>% dplyr::select(identifier,dbms,schema,operator,type) %>% unique
      ids$number=1:nrow(ids)
      filtered_data %>% left_join(ids, by = c("identifier", "dbms", "schema", "operator", "type"))  %>% dplyr::mutate(number=as.numeric(ifelse(is.na(number),1,number))) %>% ungroup %>% dplyr::mutate(number = cummax(number)) -> test
      dr_minsitrust <- test %>% dplyr::filter(generator == "directedRandom") %>% dplyr::group_by(identifier, dbms, generator, number, operator) %>% dplyr::summarise(killed_mutants = sum(killed == "true"), total_mutants = (sum(killed == "true") + sum(killed == "false")))# %>% dplyr::mutate(mutationScore = round((killed_mutants/total_mutants) * 100, 2))
      dr_itrust <- d1 %>% dplyr::filter(schema == "iTrust", generator == "directedRandom", type == "NORMAL", operator == selected_operator, dbms == db) %>% dplyr::group_by(identifier, dbms, generator, operator) %>% dplyr::summarise(killed_mutants = sum(killed == "true"), total_mutants = (sum(killed == "true") + sum(killed == "false")))# %>% dplyr::mutate(mutationScore = round((killed_mutants/total_mutants) * 100, 2))
      # Chekcing if there is any Itrust mutants
      if (nrow(dr_itrust) > 0) {
        dr_itrust$number=1:nrow(dr_itrust)
      }
      dr <- rbind(dr_minsitrust, dr_itrust)
      dr <- dr %>% dplyr::group_by(number, generator, dbms, operator) %>% dplyr::summarise(killed_mutants = sum(killed_mutants), total_mutants = sum(total_mutants)) %>% dplyr::mutate(mutationScore = round((killed_mutants/total_mutants) * 100, 2))

      # Get each run for AVM-R
      ids <- filtered_data %>% dplyr::filter(schema== first_schema[[1,1]], generator == "avs") %>% dplyr::select(identifier,dbms,schema,operator,type) %>% unique
      ids$number=1:nrow(ids)
      filtered_data %>% left_join(ids, by = c("identifier", "dbms", "schema", "operator", "type"))  %>% dplyr::mutate(number=as.numeric(ifelse(is.na(number),1,number))) %>% ungroup %>% dplyr::mutate(number = cummax(number)) -> test
      avs_minsitrust <- test %>% dplyr::filter(generator == "avs") %>% dplyr::group_by(identifier, dbms, generator, number, operator) %>% dplyr::summarise(killed_mutants = sum(killed == "true"), total_mutants = (sum(killed == "true") + sum(killed == "false")))# %>% dplyr::mutate(mutationScore = round((killed_mutants/total_mutants) * 100, 2))
      avs_itrust <- d1 %>% dplyr::filter(schema == "iTrust", generator == "avs", type == "NORMAL", operator == selected_operator, dbms == db) %>% dplyr::group_by(identifier, dbms, generator, operator) %>% dplyr::summarise(killed_mutants = sum(killed == "true"), total_mutants = (sum(killed == "true") + sum(killed == "false")))# %>% dplyr::mutate(mutationScore = round((killed_mutants/total_mutants) * 100, 2))
      # Chekcing if there is any Itrust mutants
      if (nrow(avs_itrust) > 0) {
        avs_itrust$number=1:nrow(avs_itrust)
      }
      avm <- rbind(avs_minsitrust, avs_itrust)
      avm <- avm %>% dplyr::group_by(number, generator, dbms, operator) %>% dplyr::summarise(killed_mutants = sum(killed_mutants), total_mutants = sum(total_mutants)) %>% dplyr::mutate(mutationScore = round((killed_mutants/total_mutants) * 100, 2))

      # Get each run for AVM-D
      ids <- filtered_data %>% dplyr::filter(schema== first_schema[[1,1]], generator == "avsDefaults") %>% dplyr::select(identifier,dbms,schema,operator,type) %>% unique
      ids$number=1:nrow(ids)
      filtered_data %>% left_join(ids, by = c("identifier", "dbms", "schema", "operator", "type"))  %>% dplyr::mutate(number=as.numeric(ifelse(is.na(number),1,number))) %>% ungroup %>% dplyr::mutate(number = cummax(number)) -> test
      avsd_minsitrust <- test %>% dplyr::filter(generator == "avsDefaults") %>% dplyr::group_by(identifier, dbms, generator, number, operator) %>% dplyr::summarise(killed_mutants = sum(killed == "true"), total_mutants = (sum(killed == "true") + sum(killed == "false")))# %>% dplyr::mutate(mutationScore = round((killed_mutants/total_mutants) * 100, 2))
      avsd_itrust <- d1 %>% dplyr::filter(schema == "iTrust", generator == "avsDefaults", type == "NORMAL", operator == selected_operator, dbms == db) %>% dplyr::group_by(identifier, dbms, generator, operator) %>% dplyr::summarise(killed_mutants = sum(killed == "true"), total_mutants = (sum(killed == "true") + sum(killed == "false")))# %>% dplyr::mutate(mutationScore = round((killed_mutants/total_mutants) * 100, 2))
      # Chekcing if there is any Itrust mutants
      if (nrow(avsd_itrust) > 0) {
        avsd_itrust$number=1:nrow(avsd_itrust)
      }
      avmd <- rbind(avsd_minsitrust, avsd_itrust)
      avmd <- avmd %>% dplyr::group_by(number, generator, dbms, operator) %>% dplyr::summarise(killed_mutants = sum(killed_mutants), total_mutants = sum(total_mutants)) %>% dplyr::mutate(mutationScore = round((killed_mutants/total_mutants) * 100, 2))

      # Get each run for Random
      ids <- filtered_data %>% dplyr::filter(schema== first_schema[[1,1]], generator == "random") %>% dplyr::select(identifier,dbms,schema,operator,type) %>% unique
      ids$number=1:nrow(ids)
      filtered_data %>% left_join(ids, by = c("identifier", "dbms", "schema", "operator", "type"))  %>% dplyr::mutate(number=as.numeric(ifelse(is.na(number),1,number))) %>% ungroup %>% dplyr::mutate(number = cummax(number)) -> test
      ran_minsitrust <- test %>% dplyr::filter(generator == "random") %>% dplyr::group_by(identifier, dbms, generator, number, operator) %>% dplyr::summarise(killed_mutants = sum(killed == "true"), total_mutants = (sum(killed == "true") + sum(killed == "false")))# %>% dplyr::mutate(mutationScore = round((killed_mutants/total_mutants) * 100, 2))
      ran_itrust <- d1 %>% dplyr::filter(schema == "iTrust", generator == "random", type == "NORMAL", operator == selected_operator, dbms == db) %>% dplyr::group_by(identifier, dbms, generator, operator) %>% dplyr::summarise(killed_mutants = sum(killed == "true"), total_mutants = (sum(killed == "true") + sum(killed == "false")))# %>% dplyr::mutate(mutationScore = round((killed_mutants/total_mutants) * 100, 2))
      # Chekcing if there is any Itrust mutants
      if (nrow(ran_itrust) > 0) {
        ran_itrust$number=1:nrow(avsd_itrust)
      }
      rand <- rbind(ran_minsitrust, ran_itrust)
      rand <- rand %>% dplyr::group_by(number, generator, dbms, operator) %>% dplyr::summarise(killed_mutants = sum(killed_mutants), total_mutants = sum(total_mutants)) %>% dplyr::mutate(mutationScore = round((killed_mutants/total_mutants) * 100, 2))

      # save each run per DBMS
      if (db == "Postgres") {
        postgres_dr <- dr
        postgres_avm <- avm
        postgres_avmd <- avmd
        postgres_rand <- rand
      } else if (db == "SQLite") {
        sqlite_dr <- dr
        sqlite_avm <- avm
        sqlite_avmd <- avmd
        sqlite_rand <- rand
      } else if (db == "HyperSQL") {
        hsql_dr <- dr
        hsql_avm <- avm
        hsql_avmd <- avmd
        hsql_rand <- rand
      }

    }
    # Arrange by Runs (numbers)
    postgres_dr <- dplyr::arrange(postgres_dr, number)
    postgres_avm <- dplyr::arrange(postgres_avm, number)
    postgres_avmd <- dplyr::arrange(postgres_avmd, number)
    postgres_rand <- dplyr::arrange(postgres_rand, number)
    sqlite_dr <- dplyr::arrange(sqlite_dr, number)
    sqlite_avm <- dplyr::arrange(sqlite_avm, number)
    sqlite_avmd <- dplyr::arrange(sqlite_avmd, number)
    sqlite_rand <- dplyr::arrange(sqlite_rand, number)
    hsql_dr <- dplyr::arrange(hsql_dr, number)
    hsql_avm <- dplyr::arrange(hsql_avm, number)
    hsql_avmd <- dplyr::arrange(hsql_avmd, number)
    hsql_rand <- dplyr::arrange(hsql_rand, number)

    # bind them together
    postgres <- rbind(postgres_dr, postgres_avm, postgres_avmd, postgres_rand)
    sqlite <- rbind(sqlite_dr, sqlite_avm, sqlite_avmd, sqlite_rand)
    hsql <- rbind(hsql_dr, hsql_avm, hsql_avmd, hsql_rand)

    dt <- rbind(dt, postgres, sqlite, hsql)
  }
  return(dt)
}