R/plotDataSetList.R

In IOHprofiler/IOHanalyzer: Data Analysis Part of 'IOHprofiler'

symbols <-
  c(
    "circle-open",
    "diamond-open",
    "square-open",
    "cross-open",
    "triangle-up-open",
    "triangle-down-open"
  )

get_legends <- function(dsList) {
  N <- length(dsList)
  legends <- sapply(dsList, function(d)
    get_id(d))

  if (length(unique(legends)) < N) {
    funcId <- sapply(dsList, function(d)
      attr(d, 'funcId'))
    if (length(unique(funcId)) > 1)
      legends <- paste0(legends, '-F', funcId)
  }

  if (length(unique(legends)) < N) {
    DIM <- sapply(dsList, function(d)
      attr(d, 'DIM'))
    if (length(unique(DIM)) > 1)
      legends <- paste0(legends, '-', DIM, 'D')
  }
  legends
}

insert_best_parts <- function(from_data, to_data, best_is_min) {
  if (all(is.na(from_data)))
    to_data
  else
    if (best_is_min)
      pmin(from_data, to_data, na.rm = T)
  else
    pmax(from_data, to_data, na.rm = T)
}

generate_rbga <- function(color, a) {
  paste0('rgba(', paste0(color, collapse = ','), ',', a, ')')
}

grad_functions <- c(
  scaled_edges = function(count, amount, intensity) {
    scale <- (intensity + 1) / 2
    color_end <- floor(scale * amount * 2)
    if (count < color_end)
      1 / color_end
    else
      0
  }
  ,
  fixed_edges = function(count, amount, intensity) {
    scale <- (intensity + 1) / 2
    color_center <- floor(scale * amount) + 1
    if (count <= color_center)
      1 / (2 * color_center)
    else
      1 / (2 * (amount - color_center))
  }
)

#S3 generics
# TODO: decide which parameters need to be in the generics

#' Plot lineplot of the ERTs of a DataSetList
#'
#' @param dsList A DataSetList (should consist of only one function and dimension).
#' @param Fstart The starting function value.
#' @param Fstop The final function value.
#' @param show.ERT Whether or not to show the ERT-values
#' @param show.CI Whether or not to show the standard deviations
#' @param show.mean Whether or not to show the mean hitting times
#' @param show.median Whether or not to show the median hitting times
#' @param scale.xlog Whether or not to scale the x-axis logaritmically
#' @param scale.ylog Whether or not to scale the y-axis logaritmically
#' @param scale.reverse Wheter or not to reverse the x-axis (when using minimization)
#' @param backend Which plotting library to use. Can be 'plotly' or 'ggplot2'
#' @param includeOpts Whether or not to include all best points reached by each algorithm
#' @param p Existing plot to which to add the current data
#' @return A plot of ERT-values of the DataSetList
#' @export
#' @examples
#' Plot.RT.Single_Func(subset(dsl, funcId == 1))
Plot.RT.Single_Func <- function(dsList,
                                Fstart = NULL,
                                Fstop = NULL,
                                show.ERT = T,
                                show.CI = F,
                                show.mean = F,
                                show.median = F,
                                backend = NULL,
                                scale.xlog = F,
                                scale.ylog = F,
                                scale.reverse = F,
                                includeOpts = F,
                                p = NULL)
UseMethod("Plot.RT.Single_Func", dsList)
#' Plot lineplot of the expected function values of a DataSetList
#'
#' @param dsList A DataSetList (should consist of only one function and dimension).
#' @param RTstart The starting runtime value.
#' @param RTstop The final runtime value.
#' @param show.CI Whether or not to show the standard deviations
#' @param show.mean Whether or not to show the mean runtimes
#' @param show.median Whether or not to show the median runtimes
#' @param scale.xlog Whether or not to scale the x-axis logaritmically
#' @param scale.ylog Whether or not to scale the y-axis logaritmically
#' @param scale.reverse Wheter or not to reverse the x-axis (when using minimization)
#' @param backend Which plotting library to use. Can be 'plotly' or 'ggplot2'
#'
#' @return A plot of ERT-values of the DataSetList
#' @export
#' @examples
#' Plot.FV.Single_Func(subset(dsl, funcId == 1))
Plot.FV.Single_Func <-
  function(dsList,
           RTstart = NULL,
           RTstop = NULL,
           show.CI = F,
           show.mean = T,
           show.median = F,
           backend = NULL,
           scale.xlog = F,
           scale.ylog = F,
           scale.reverse = F)
    UseMethod("Plot.FV.Single_Func", dsList)
#' Plot probablity mass function of the runtimes of a DataSetList at a certain target function value
#'
#' @param dsList A DataSetList (should consist of only one function and dimension).
#' @param ftarget The target function value.
#' @param show.sample Whether or not to show the individual runtime samples
#' @param scale.ylog Whether or not to scale the y-axis logaritmically
#' @param backend Which plotting library to use. Can be 'plotly' or 'ggplot2'
#'
#' @return A plot of the probablity mass function of the runtimes at a the
#'         target function value of the DataSetList
#' @export
#' @examples
#' Plot.RT.PMF(subset(dsl, funcId == 1), 14)
Plot.RT.PMF <-
  function(dsList,
           ftarget,
           show.sample = F,
           scale.ylog = F,
           backend = NULL)
    UseMethod("Plot.RT.PMF", dsList)
#' Plot histograms of the runtimes of a DataSetList at a certain target function value
#'
#' @param dsList A DataSetList (should consist of only one function and dimension).
#' @param ftarget The target function value.
#' @param plot_mode How to plot the different hisograms for each algorithm. Can be either
#'  'overlay' to show all algorithms on one plot, or 'subplot' to have one plot per algorithm.
#' @param use.equal.bins Whether to determine one bin size for all plots or have individual
#' bin sizes for each algorithm
#'
#' @return A plot of the histograms of the runtimes at a the
#'         target function value of the DataSetList
#' @export
#' @examples
#' Plot.RT.Histogram(subset(dsl, funcId == 1), 14)
Plot.RT.Histogram <-
  function(dsList,
           ftarget,
           plot_mode = 'overlay',
           use.equal.bins = F)
    UseMethod("Plot.RT.Histogram", dsList)
#' Plot the empirical cumulative distriburtion as a function of the running times of
#' a DataSetList at certain target function values
#'
#' @param dsList A DataSetList (should consist of only one function and dimension).
#' @param ftargets The target function values
#' @param scale.xlog Whether or not to scale the x-axis logaritmically
#'
#' @return A plot of the empirical cumulative distriburtion as a function of
#' the running times of the DataSetList at the target function values
#' @export
#' @examples
#' \dontshow{data.table::setDTthreads(1)}
#' Plot.RT.ECDF_Per_Target(subset(dsl, funcId == 1), 14)
Plot.RT.ECDF_Per_Target <-
  function(dsList, ftargets, scale.xlog = F)
    UseMethod("Plot.RT.ECDF_Per_Target", dsList)
#' Plot the aggregated empirical cumulative distriburtion as a function of the running times of
#' a DataSetList.
#'
#' @param dsList A DataSetList (should consist of only one function and dimension).
#' @param fstart The starting function value
#' @param fstop The final function value
#' @param fstep The spacing between starting and final function values
#' @param show.per_target Whether or not to show the individual ECDF-curves for each target
#' @param scale.xlog Whether or not to scale the x-axis logaritmically
#'
#' @return A plot of the empirical cumulative distriburtion as a function of
#' the running times of the DataSetList
#' @export
#' @examples
#' \dontshow{data.table::setDTthreads(1)}
#' Plot.RT.ECDF_Single_Func(subset(dsl, funcId == 1))
Plot.RT.ECDF_Single_Func <-
  function(dsList,
           fstart = NULL,
           fstop = NULL,
           fstep = NULL,
           show.per_target = F,
           scale.xlog = F)
    UseMethod("Plot.RT.ECDF_Single_Func", dsList)
#' Radarplot of the area under the aggregated ECDF-curve of a DataSetList.
#'
#' @param dsList A DataSetList (should consist of only one function and dimension).
#' @param fstart The starting function value
#' @param fstop The final function value
#' @param fstep The spacing between starting and final function values
#' @param fval_formatter Function to format the function-value labels
#'
#' @return A radarplot of the area under the aggregated ECDF-curve of the DataSetList
#' @export
#' @examples
#' \dontshow{data.table::setDTthreads(1)}
#' Plot.RT.ECDF_AUC(subset(dsl, funcId == 1))
Plot.RT.ECDF_AUC <- function(dsList,
                             fstart = NULL,
                             fstop = NULL,
                             fstep = NULL,
                             fval_formatter = as.integer)
  UseMethod("Plot.RT.ECDF_AUC", dsList)
#' Plot probablity density function of the function values of a DataSetList at
#' a certain target runtime
#'
#' @param dsList A DataSetList (should consist of only one function and dimension).
#' @param runtime The target runtime
#' @param show.sample Whether or not to show the individual function value samples
#' @param scale.ylog Whether or not to scale the y-axis logaritmically
#'
#' @return A plot of the probablity density function of the runtimes at a the
#'         target function value of the DataSetList
#' @export
#' @examples
#' Plot.FV.PDF(subset(dsl, funcId == 1), 100)
Plot.FV.PDF <-
  function(dsList,
           runtime,
           show.sample = F,
           scale.ylog = F)
    UseMethod("Plot.FV.PDF", dsList)
#' Plot histograms of the function values of a DataSetList at a certain target runtime
#'
#' @param dsList A DataSetList (should consist of only one function and dimension).
#' @param runtime The target runtime
#' @param plot_mode How to plot the different hisograms for each algorithm. Can be either
#'  'overlay' to show all algorithms on one plot, or 'subplot' to have one plot per algorithm.
#' @param use.equal.bins Whether to determine one bin size for all plots or have individual
#' bin sizes for each algorithm
#'
#' @return A plot of the histograms of the function values at a the
#'         target runtime of the DataSetList
#' @export
#' @examples
#' Plot.FV.Histogram(subset(dsl, funcId == 1), 100)
Plot.FV.Histogram <-
  function(dsList,
           runtime,
           plot_mode = 'overlay',
           use.equal.bins = F)
    UseMethod("Plot.FV.Histogram", dsList)
#' Plot the empirical cumulative distriburtion as a function of the target values of
#' a DataSetList at certain target runtimes
#'
#' @param dsList A DataSetList (should consist of only one function and dimension).
#' @param runtimes The target runtimes
#' @param scale.xlog Whether or not to scale the x-axis logaritmically
#' @param scale.reverse Whether or not to reverse the x-axis (when using minimization)
#'
#' @return A plot of the empirical cumulative distriburtion as a function of
#' the fucntion values of the DataSetList at the target runtimes
#' @export
#' @examples
#' \dontshow{data.table::setDTthreads(1)}
#' Plot.FV.ECDF_Per_Target(subset(dsl, funcId == 1), 10)
Plot.FV.ECDF_Per_Target <-
  function(dsList,
           runtimes,
           scale.xlog = F,
           scale.reverse = F)
    UseMethod("Plot.FV.ECDF_Per_Target", dsList)
#' Plot the aggregated empirical cumulative distriburtion as a function of the function values of
#' a DataSetList.
#'
#' @param dsList A DataSetList (should consist of only one function and dimension).
#' @param rt_min The starting runtime
#' @param rt_max The final runtime
#' @param rt_step The spacing between starting and final runtimes
#' @param show.per_target Whether or not to show the individual ECDF-curves for each runtime
#' @param scale.xlog Whether or not to scale the x-axis logaritmically
#' @param scale.reverse Whether or not to reverse the x-axis (when using minimization)
#'
#' @return A plot of the empirical cumulative distriburtion as a function of
#' the function values of the DataSetList
#' @export
#' @examples
#' \dontshow{data.table::setDTthreads(1)}
#' Plot.FV.ECDF_Single_Func(subset(dsl, funcId == 1))
Plot.FV.ECDF_Single_Func <-
  function(dsList,
           rt_min = NULL,
           rt_max = NULL,
           rt_step = NULL,
           scale.xlog = F,
           show.per_target = F,
           scale.reverse = F)
    UseMethod("Plot.FV.ECDF_Single_Func", dsList)
#' Radarplot of the area under the aggregated ECDF-curve of a DataSetList.
#'
#' @param dsList A DataSetList (should consist of only one function and dimension).
#' @param rt_min The starting runtime
#' @param rt_max The final runtime
#' @param rt_step The spacing between starting and final runtimes
#'
#' @return A radarplot of the area under the aggregated ECDF-curve of the DataSetList
#' @export
#' @examples
#' \dontshow{data.table::setDTthreads(1)}
#' Plot.FV.ECDF_AUC(subset(dsl, funcId == 1))
Plot.FV.ECDF_AUC <- function(dsList,
                             rt_min = NULL,
                             rt_max = NULL,
                             rt_step = NULL)
  UseMethod("Plot.FV.ECDF_AUC", dsList)
#' Plot the parameter values recorded in a DataSetList (aligned by funcion value)
#'
#' @param dsList A DataSetList (should consist of only one function and dimension).
#' @param f_min The starting function value.
#' @param f_max The final function value.
#' @param show.mean Whether or not to show the mean parameter values
#' @param show.median Whether or not to show the median parameter values
#' @param scale.xlog Whether or not to scale the x-axis logaritmically
#' @param scale.ylog Whether or not to scale the y-axis logaritmically
#' @param algids Which algorithms from dsList to use
#' @param par_name Which parameters to create plots for; set to NULL to use all
#' parameters found in dsList.
#' @param show.CI Whether or not to show the standard deviation
#'
#' @return A plot of for every recorded parameter in the DataSetList
#' @export
#' @examples
#' Plot.RT.Parameters(subset(dsl, funcId == 1))
Plot.RT.Parameters <- function(dsList,
                               f_min = NULL,
                               f_max = NULL,
                               algids = 'all',
                               par_name = NULL,
                               scale.xlog = F,
                               scale.ylog = F,
                               show.mean = T,
                               show.median = F,
                               show.CI = F)
  UseMethod("Plot.RT.Parameters", dsList)
#' Plot the parameter values recorded in a DataSetList (aligned by budget)
#'
#' @param dsList A DataSetList (should consist of only one function and dimension).
#' @param rt_min The starting budget value.
#' @param rt_max The final budget value.
#' @param show.mean Whether or not to show the mean parameter values
#' @param show.median Whether or not to show the median parameter values
#' @param scale.xlog Whether or not to scale the x-axis logaritmically
#' @param scale.ylog Whether or not to scale the y-axis logaritmically
#' @param algids Which algorithms from dsList to use
#' @param par_name Which parameters to create plots for; set to NULL to use all
#' parameters found in dsList.
#' @param show.CI Whether or not to show the standard deviation
#'
#' @return A plot of for every recorded parameter in the DataSetList
#' @export
#' @examples
#' Plot.FV.Parameters(subset(dsl, funcId == 1))
Plot.FV.Parameters <- function(dsList,
                               rt_min = NULL,
                               rt_max = NULL,
                               algids = 'all',
                               par_name = NULL,
                               scale.xlog = F,
                               scale.ylog = F,
                               show.mean = T,
                               show.median = F,
                               show.CI = F)
  UseMethod("Plot.FV.Parameters", dsList)
#' Plot the aggregated empirical cumulative distriburtion as a function of the running times of
#' a DataSetList. Aggregated over multiple functions or dimensions.
#'
#' @param dsList A DataSetList.
#' @param targets The target function values. Specified in a data.frame, as can be generated
#' @param scale.xlog Whether or not to scale the x-axis logaritmically
#' by the function 'get_ECDF_targets'
#'
#' @return A plot of the empirical cumulative distriburtion as a function of
#' the running times of the DataSetList
#' @export
#' @examples
#' \dontshow{data.table::setDTthreads(1)}
#' Plot.RT.ECDF_Multi_Func(dsl)
Plot.RT.ECDF_Multi_Func <-
  function(dsList,
           targets = NULL,
           scale.xlog = F)
    UseMethod("Plot.RT.ECDF_Multi_Func", dsList)
#' Plot ERT-plots for multiple functions or dimensions
#'
#' @param dsList A DataSetList (should consist of only one function OR dimension).
#' @param scale.xlog Whether or not to scale the x-axis logaritmically
#' @param scale.ylog Whether or not to scale the y-axis logaritmically
#' @param scale.reverse Wheter or not to reverse the x-axis (when using minimization)
#' @param backend Which plotting library to use. Either 'plotly' or 'ggplot2'.
#'
#' @return A plot of ERT-values of the DataSetList
#' @export
#' @examples
#' Plot.RT.Multi_Func(dsl)
Plot.RT.Multi_Func <-
  function(dsList,
           scale.xlog = F,
           scale.ylog = F,
           scale.reverse = F,
           backend = NULL)
    UseMethod("Plot.RT.Multi_Func", dsList)
#' Plot ERT-based comparison over multiple functions or dimensions
#'
#' @param dsList A DataSetList (should consist of only one function OR dimension).
#' @param plot_mode How the plots should be created. Can be 'line' or 'radar'
#' @param aggr_on Whether to compare on functions ('funcId') or dimensions ('DIM')
#' @param scale.ylog Whether or not to scale the y-axis logaritmically
#' @param maximize Wheter or not to the data is of a maximization problem
#' @param targets Custom list of function-value targets, one for each function or dimension.
#' @param use_rank Wheter to use a ranking system. If False, the actual ERT-values will be used.
#' @param erts Pre-calculated ERT-values for the provided targets. Created by the max_ERTs function
#' of DataSetList. Can be provided to prevent needless computation in recalculating ERTs when recreating
#' this plot.
#' @param inf.action How to handle infinite ERTs ('overlap' or 'jitter')
#' @return A plot of ERT-based comparison on the provided functions or dimensions of the DataSetList
#' @export
#' @examples
#' Plot.RT.Aggregated(dsl)
Plot.RT.Aggregated <-
  function(dsList,
           aggr_on = 'funcId',
           targets = NULL,
           plot_mode = 'radar',
           use_rank = F,
           scale.ylog = T,
           maximize = T,
           erts = NULL,
           inf.action = 'overlap')
    UseMethod("Plot.RT.Aggregated", dsList)
#' Plot expected function value-based comparison over multiple functions or dimensions
#'
#' @param dsList A DataSetList (should consist of only one function OR dimension).
#' @param plot_mode How the plots should be created. Can be 'line' or 'radar'
#' @param aggr_on Whether to compare on functions ('funcId') or dimensions ('DIM')
#' @param scale.ylog Whether or not to scale the y-axis logaritmically
#' @param runtimes Custom list of function-value targets, one for each function or dimension.
#' @param use_rank Wheter to use a ranking system. If False, the actual expected function-
#' values will be used.
#' @param fvs Pre-calculated expected function-values for the provided runtimes Created by the
#' max_ERTs function of DataSetList. Can be provided to prevent needless computation
#' in recalculating ERTs when recreating this plot.
#'
#' @return A plot of expected function value-based comparison on the provided functions
#'  or dimensions of the DataSetList
#' @export
#' @examples
#' Plot.FV.Aggregated(dsl)
Plot.FV.Aggregated <-
  function(dsList,
           aggr_on = 'funcId',
           runtimes = NULL,
           plot_mode = 'radar',
           use_rank = F,
           scale.ylog = T,
           fvs = NULL)
    UseMethod("Plot.FV.Aggregated", dsList)

#' Plot FV-plots for multiple functions or dimensions
#'
#' @param dsList A DataSetList (should consist of only one function OR dimension).
#' @param scale.xlog Whether or not to scale the x-axis logaritmically
#' @param scale.ylog Whether or not to scale the y-axis logaritmically
#' @param backend Which plotting library to use. Either 'plotly' or 'ggplot2'.
#'
#' @return A plot of Function-values of the DataSetList
#' @export
#' @examples
#' Plot.FV.Multi_Func(dsl)
Plot.FV.Multi_Func <-
  function(dsList,
           scale.xlog = F,
           scale.ylog = F,
           backend = NULL)
    UseMethod("Plot.FV.Multi_Func", dsList)

#' Plot a heatmap showing the statistically different algorithms
#'
#' @param dsList A DataSetList (should consist of only one function and dimension).
#' @param ftarget The target function value to use
#' @param alpha The cutoff for statistical significance
#' @param bootstrap.size The amound of bootstrapped samples used
#' @param which Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective
#'
#' @return A heatmap showing the statistical significance between algorithms
#' @export
#' @examples
#' Plot.Stats.Significance_Heatmap(subset(dsl, funcId == 2), 16)
Plot.Stats.Significance_Heatmap <-
  function(dsList,
           ftarget,
           alpha = 0.01,
           bootstrap.size = 30,
           which = 'by_FV')
    UseMethod("Plot.Stats.Significance_Heatmap", dsList)

#' Plot a heatmap according to the specifications from the Nevergrad dashboard
#'
#' @param dsList A DataSetList (should consist of only one function and dimension).
#' @param target_dt A data-table containing the targets to condider on each function/dimension pair
#' @param which Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective
#'
#' @return A heatmap showing the fraction of times algorithm A beats algorithm B
#' @export
#' @examples
#' Plot.Comparison.Heatmap(dsl)
Plot.Comparison.Heatmap <-
  function(dsList, target_dt, which = 'by_FV')
    UseMethod("Plot.Comparison.Heatmap", dsList)

#' Plot a network graph showing the statistically different algorithms
#'
#' @param dsList A DataSetList (should consist of only one function and dimension).
#' @param ftarget The target function value to use
#' @param alpha The cutoff for statistical significance
#' @param bootstrap.size The amound of bootstrapped samples used
#' @param which Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective
#'
#' @return A graph showing the statistical significance between algorithms
#' @export
#' @examples
#' Plot.Stats.Significance_Graph(subset(dsl, funcId == 2), 16)
Plot.Stats.Significance_Graph <-
  function(dsList,
           ftarget,
           alpha = 0.01,
           bootstrap.size = 30,
           which = 'by_FV')
    UseMethod("Plot.Stats.Significance_Graph", dsList)

#' Create a candlestick plot of Glicko2-rankings
#'
#' @param dsList A DataSetList
#' @param nr_rounds The number of rounds in the tournament
#' @param glicko2_rank_df Optional. Dataframe containing the glicko2 rating to avoid needless recalculation.
#' @param which Whether to use fixed-target ('by_FV') or fixed-budget ('by_RT') perspective
#' @param target_dt Optional: data table containing the targets for each function and dimension
#'
#' @export
#' @examples
#' Plot.Stats.Glicko2_Candlestick(dsl, nr_rounds=2)
Plot.Stats.Glicko2_Candlestick <-
  function(dsList,
           nr_rounds = 100,
           glicko2_rank_df = NULL,
           which = 'by_FV',
           target_dt = NULL)
    UseMethod("Plot.Stats.Glicko2_Candlestick", dsList)


##Implementations

#' @rdname Plot.RT.Single_Func
#' @export
Plot.RT.Single_Func.DataSetList <-
  function(dsList,
           Fstart = NULL,
           Fstop = NULL,
           show.ERT = T,
           show.CI = T,
           show.mean = F,
           show.median = F,
           backend = NULL,
           scale.xlog = F,
           scale.ylog = F,
           scale.reverse = F,
           includeOpts = F,
           p = NULL) {
    if (is.null(backend))
      backend <-
        getOption("IOHanalyzer.backend", default = 'plotly')

    if (backend == 'plotly') {
      data <-
        generate_data.Single_Function(dsList, Fstart, Fstop, scale.xlog, 'by_RT', includeOpts)

      y_attrs <- c()
      if (show.ERT)
        y_attrs <- c(y_attrs, 'ERT')
      if (show.mean)
        y_attrs <- c(y_attrs, 'mean')
      if (show.median)
        y_attrs <- c(y_attrs, 'median')
      show_legend <- T
      if (length(y_attrs) > 0) {
        p <- plot_general_data(
          data,
          x_attr = 'target',
          y_attr = y_attrs,
          type = 'line',
          legend_attr = 'ID',
          show.legend = show_legend,
          scale.ylog = scale.ylog,
          p = p,
          scale.xlog = scale.xlog,
          x_title = "Best-so-far f(x)-value",
          y_title = "Function Evaluations",
          scale.reverse = scale.reverse
        )
        show_legend <- F
      }
      if (show.CI) {
        p <- plot_general_data(
          data,
          x_attr = 'target',
          y_attr = 'mean',
          type = 'ribbon',
          legend_attr = 'ID',
          lower_attr = 'lower',
          upper_attr = 'upper',
          p = p,
          show.legend = show_legend,
          scale.ylog = scale.ylog,
          scale.xlog = scale.xlog,
          x_title = "Best-so-far f(x)-value",
          y_title = "Function Evaluations",
          scale.reverse = scale.reverse
        )
      }
    }
    # } else if (backend == 'ggplot2') {
    #   dt[, 'group' := paste(algId, funcId, DIM, sep = '-')]
    #   p <- ggplot(data = dt, aes(group = `group`, colour = `group`))
    #
    #   if (show.CI) p <- p + geom_ribbon(aes(target, ymin = lower, ymax = upper, fill = `group`),
    #                                     alpha = 0.2, colour = NA)
    #   if (show.ERT) p <- p + geom_line(aes(target, ERT), size = 1.2)
    #   if (show.mean) p <- p + geom_line(aes(target, mean), linetype = 'dashed')
    #   if (show.median) p <- p + geom_line(aes(target, median), linetype = 'dotted')
    #
    #   p <- p +
    #     scale_color_manual(values = colors) +
    #     scale_fill_manual(values = colors)
    # }
    return(p)
  }

#' @rdname Plot.FV.Single_Func
#' @export
Plot.FV.Single_Func.DataSetList <-
  function(dsList,
           RTstart = NULL,
           RTstop = NULL,
           show.CI = F,
           show.mean = T,
           show.median = F,
           backend = NULL,
           scale.xlog = F,
           scale.ylog = F,
           scale.reverse = F) {
    if (is.null(backend))
      backend <-
        getOption("IOHanalyzer.backend", default = 'plotly')

    if (backend == 'plotly') {
      data <-
        generate_data.Single_Function(dsList, RTstart, RTstop, scale.xlog, 'by_FV')

      y_attrs <- c()
      if (show.mean)
        y_attrs <- c(y_attrs, 'mean')
      if (show.median)
        y_attrs <- c(y_attrs, 'median')
      show_legend <- T
      if (length(y_attrs) > 0) {
        p <- plot_general_data(
          data,
          x_attr = 'runtime',
          y_attr = y_attrs,
          type = 'line',
          legend_attr = 'ID',
          show.legend = show_legend,
          scale.ylog = scale.ylog,
          scale.xlog = scale.xlog,
          x_title = "Best-so-far f(x)-value",
          y_title = "Function Evaluations",
          scale.reverse = scale.reverse
        )
        show_legend <- F
      }
      else
        p <- NULL
      if (show.CI) {
        p <- plot_general_data(
          data,
          x_attr = 'runtime',
          y_attr = 'mean',
          type = 'ribbon',
          legend_attr = 'ID',
          lower_attr = 'lower',
          upper_attr = 'upper',
          p = p,
          show.legend = show_legend,
          scale.ylog = scale.ylog,
          scale.xlog = scale.xlog,
          x_title = "Best-so-far f(x)-value",
          y_title = "Function Evaluations",
          scale.reverse = scale.reverse
        )
      }

    }
    # } else if (backend == 'ggplot2') {
    #   fce[, 'group' := paste(algId, funcId, DIM, sep = '-')]
    #   p <- ggplot(data = fce, aes(group = `group`, colour = `group`))
    #
    #   if (show.mean) p <- p + geom_line(aes(runtime, mean), linetype = 'dashed')
    #   if (show.median) p <- p + geom_line(aes(runtime, median), linetype = 'dotted')
    #
    #   p <- p +
    #     scale_color_manual(values = colors) +
    #     scale_fill_manual(values = colors)
    #
    #   #TODO: add individual run etc
    # }
    return(p)
  }

#' @rdname Plot.RT.PMF
#' @export
Plot.RT.PMF.DataSetList <-
  function(dsList,
           ftarget,
           show.sample = F,
           scale.ylog = F,
           backend = NULL) {
    if (is.null(backend))
      backend <-
        getOption("IOHanalyzer.backend", default = 'plotly')

    data <- generate_data.PMF(dsList, ftarget, 'by_RT')

    plot_general_data(
      data,
      'ID',
      'RT',
      scale.ylog = scale.ylog,
      x_title = "Algorithm",
      y_title = "Function Evaluations"
    )
  }

#' @rdname Plot.RT.Histogram
#' @export
Plot.RT.Histogram.DataSetList <-
  function(dsList,
           ftarget,
           plot_mode = 'overlay',
           use.equal.bins = F) {
    if (length(get_funcId(dsList)) != 1 ||
        length(get_dim(dsList)) != 1) {
      warning(
        "Invalid dataset uploaded. Please ensure the datasetlist contains data
            from only one function and only one dimension."
      )
      return(NULL)
    }
    data <-
      generate_data.hist(dsList, ftarget, use.equal.bins, 'by_RT')

    subplot_attr <- if (plot_mode == 'subplot')
      'ID'
    else
      NULL
    plot_general_data(
      data,
      'x',
      'y',
      width = 'width',
      type = 'hist',
      subplot_attr = subplot_attr,
      x_title = "Function Evaluations",
      y_title = "Runs"
    )
  }

#' @rdname Plot.RT.ECDF_Per_Target
#' @export
Plot.RT.ECDF_Per_Target.DataSetList <-
  function(dsList, ftargets, scale.xlog = F) {
    req(length(ftargets) != 0)
    data <- generate_data.ECDF(dsList, ftargets, scale.xlog)
    plot_general_data(
      data,
      'x',
      'mean',
      'line',
      x_title = "Function Evaluations",
      y_title = "Proportion of runs",
      scale.xlog = scale.xlog,
      show.legend = T
    )
  }

#' @rdname Plot.RT.ECDF_Single_Func
#' @export
Plot.RT.ECDF_Single_Func.DataSetList <-
  function(dsList,
           fstart = NULL,
           fstop = NULL,
           fstep = NULL,
           show.per_target = F,
           scale.xlog = F) {
    targets <- seq_FV(get_funvals(dsList), fstart, fstop, fstep)
    req(targets)

    data <- generate_data.ECDF(dsList, targets, scale.xlog)

    plot_general_data(
      data,
      'x',
      'mean',
      'line',
      x_title = "Function Evaluations",
      y_title = "Proportion of (run, target) pairs",
      scale.xlog = scale.xlog,
      show.legend = T
    )
  }

#' @rdname Plot.RT.ECDF_AUC
#' @export
Plot.RT.ECDF_AUC.DataSetList <- function(dsList,
                                         fstart = NULL,
                                         fstop = NULL,
                                         fstep = NULL,
                                         fval_formatter = as.integer) {
  targets <-
    seq_FV(get_funvals(dsList), fstart, fstop, fstep, length.out = 10)
  req(targets)

  data <- generate_data.AUC(dsList, targets, multiple_x = TRUE)

  plot_general_data(data, 'x', 'auc', 'radar')
}

#' @rdname Plot.FV.PDF
#' @export
Plot.FV.PDF.DataSetList <-
  function(dsList,
           runtime,
           show.sample = F,
           scale.ylog = F) {
    data <- generate_data.PMF(dsList, runtime, 'by_FV')

    plot_general_data(
      data,
      'ID',
      'f(x)',
      scale.ylog = scale.ylog,
      x_title = "Algorithm",
      y_title = "Target Value"
    )
  }

#' @rdname Plot.FV.Histogram
#' @export
Plot.FV.Histogram.DataSetList <-
  function(dsList,
           runtime,
           plot_mode = 'overlay',
           use.equal.bins = F) {
    if (length(get_funcId(dsList)) != 1 ||
        length(get_dim(dsList)) != 1) {
      warning(
        "Invalid dataset uploaded. Please ensure the datasetlist contains data
            from only one function and only one dimension."
      )
      return(NULL)
    }
    data <-
      generate_data.hist(dsList, runtime, use.equal.bins, 'by_FV')

    subplot_attr <- if (plot_mode == 'subplot')
      'ID'
    else
      NULL
    plot_general_data(
      data,
      'x',
      'y',
      width = 'width',
      type = 'hist',
      subplot_attr = subplot_attr,
      x_title = "Target Values",
      y_title = "Runs"
    )
  }

#' @rdname Plot.FV.ECDF_Per_Target
#' @export
Plot.FV.ECDF_Per_Target.DataSetList <-
  function(dsList,
           runtimes,
           scale.xlog = F,
           scale.reverse = F) {
    #TODO: Fvals in legend need to be formatted properly
    runtimes <- runtimes[!is.na(runtimes)]
    req(length(runtimes) != 0)

    data <-
      generate_data.ECDF(dsList, runtimes, scale.xlog, which = 'by_FV')

    plot_general_data(
      data,
      'x',
      'mean',
      'line',
      x_title = "Target Value",
      y_title = "Proportion of runs",
      scale.xlog = scale.xlog,
      show.legend = T,
      scale.reverse = scale.reverse
    )
  }

#' @rdname Plot.FV.ECDF_Single_Func
#' @export
Plot.FV.ECDF_Single_Func.DataSetList <-
  function(dsList,
           rt_min = NULL,
           rt_max = NULL,
           rt_step = NULL,
           scale.xlog = F,
           show.per_target = F,
           scale.reverse = F) {
    targets <- seq_RT(get_funvals(dsList), rt_min, rt_max, rt_step)
    req(targets)
    data <-
      generate_data.ECDF(dsList, targets, scale.xlog, which = 'by_FV')

    plot_general_data(
      data,
      'x',
      'mean',
      'line',
      x_title = "Target Value",
      y_title = "Proportion of (run, target) pairs",
      scale.xlog = scale.xlog,
      scale.reverse = scale.reverse,
      show.legend = T
    )
  }

#' @rdname Plot.FV.ECDF_AUC
#' @export
Plot.FV.ECDF_AUC.DataSetList <-
  function(dsList,
           rt_min = NULL,
           rt_max = NULL,
           rt_step = NULL) {
    targets <-
      seq_RT(get_runtimes(dsList), rt_min, rt_max, rt_step, length.out = 10)
    req(targets)
    data <-
      generate_data.AUC(dsList, targets, which = 'by_FV', multiple_x = TRUE)

    plot_general_data(data, 'x', 'auc', 'radar')
  }

#' @rdname Plot.RT.Parameters
#' @export
Plot.RT.Parameters.DataSetList <-
  function(dsList,
           f_min = NULL,
           f_max = NULL,
           algids = 'all',
           par_name = NULL,
           scale.xlog = F,
           scale.ylog = F,
           show.mean = T,
           show.median = F,
           show.CI = F) {
    data <-
      generate_data.Parameters(dsList, scale.xlog, which = 'by_FV')

    y_attrs <- c()
    if (show.mean)
      y_attrs <- c(y_attrs, 'mean')
    if (show.median)
      y_attrs <- c(y_attrs, 'median')
    show_legend <- T
    if (length(y_attrs) > 0) {
      p <- plot_general_data(
        data,
        x_attr = 'target',
        y_attr = y_attrs,
        type = 'line',
        legend_attr = 'ID',
        show.legend = show_legend,
        scale.ylog = scale.ylog,
        subplot_attr = 'parId',
        scale.xlog = scale.xlog
      )
      show_legend <- F
    }
    else
      p <- NULL
    if (show.CI) {
      p <- plot_general_data(
        data,
        x_attr = 'target',
        y_attr = 'mean',
        type = 'ribbon',
        legend_attr = 'ID',
        lower_attr = 'lower',
        upper_attr = 'upper',
        p = p,
        show.legend = show_legend,
        scale.ylog = scale.ylog,
        subplot_attr = 'parId',
        scale.xlog = scale.xlog
      )
    }
    p
  }


#' @rdname Plot.FV.Parameters
#' @export
Plot.FV.Parameters.DataSetList <-
  function(dsList,
           rt_min = NULL,
           rt_max = NULL,
           algids = 'all',
           par_name = NULL,
           scale.xlog = F,
           scale.ylog = F,
           show.mean = T,
           show.median = F,
           show.CI = F) {
    data <-
      generate_data.Parameters(dsList, scale.xlog, which = 'by_RT')

    y_attrs <- c()
    if (show.mean)
      y_attrs <- c(y_attrs, 'mean')
    if (show.median)
      y_attrs <- c(y_attrs, 'median')
    show_legend <- T
    if (length(y_attrs) > 0) {
      p <- plot_general_data(
        data,
        x_attr = 'runtime',
        y_attr = y_attrs,
        type = 'line',
        legend_attr = 'ID',
        show.legend = show_legend,
        scale.ylog = scale.ylog,
        subplot_attr = 'parId',
        scale.xlog = scale.xlog
      )
      show_legend <- F
    }
    else
      p <- NULL
    if (show.CI) {
      p <- plot_general_data(
        data,
        x_attr = 'runtime',
        y_attr = 'mean',
        type = 'ribbon',
        legend_attr = 'ID',
        lower_attr = 'lower',
        upper_attr = 'upper',
        p = p,
        show.legend = show_legend,
        scale.ylog = scale.ylog,
        subplot_attr = 'parId',
        scale.xlog = scale.xlog
      )
    }
    p
  }

#' @rdname Plot.RT.ECDF_Multi_Func
#' @export
Plot.RT.ECDF_Multi_Func.DataSetList <-
  function(dsList,
           targets = NULL,
           scale.xlog = F) {
    if (is.null(targets) || !is.data.table(targets)) {
      targets <- get_ECDF_targets(dsList)
    }

    data <- generate_data.ECDF(dsList, targets, scale.xlog)

    plot_general_data(
      data,
      'x',
      'mean',
      'line',
      scale.xlog = scale.xlog,
      x_title = "Function Evaluations",
      y_title = "Proportion of (run, target, ...) pairs",
      show.legend = T
    )
  }

#' @rdname Plot.RT.Multi_Func
#' @export
Plot.RT.Multi_Func.DataSetList <- function(dsList,
                                           scale.xlog = F,
                                           scale.ylog = F,
                                           scale.reverse = F,
                                           backend = NULL) {
  if (is.null(backend))
    backend <- getOption("IOHanalyzer.backend", default = 'plotly')

  data <- rbindlist(lapply(get_funcId(dsList), function(fid) {
    generate_data.Single_Function(subset(dsList, funcId == fid),
                                  scale_log = scale.xlog,
                                  which = 'by_RT')
  }))

  plot_general_data(
    data,
    x_attr = 'target',
    y_attr = 'ERT',
    subplot_attr = 'funcId',
    type = 'line',
    scale.xlog = scale.xlog,
    scale.ylog = scale.ylog,
    x_title = 'Best-so-far f(x)',
    y_title = 'ERT',
    show.legend = T,
    scale.reverse = scale.reverse
  )
}

#' @rdname Plot.FV.Multi_Func
#' @export
Plot.FV.Multi_Func.DataSetList <- function(dsList,
                                           scale.xlog = F,
                                           scale.ylog = F,
                                           backend = NULL) {
  if (is.null(backend))
    backend <- getOption("IOHanalyzer.backend", default = 'plotly')

  data <- rbindlist(lapply(get_funcId(dsList), function(fid) {
    generate_data.Single_Function(subset(dsList, funcId == fid),
                                  scale_log = scale.xlog,
                                  which = 'by_FV')
  }))

  plot_general_data(
    data,
    x_attr = 'runtime',
    y_attr = 'mean',
    subplot_attr = 'funcId',
    type = 'line',
    scale.xlog = scale.xlog,
    scale.ylog = scale.ylog,
    x_title = 'Runtime',
    y_title = 'Best-so-far f(x)',
    show.legend = T
  )
}

#' @rdname Plot.RT.Aggregated
#' @export
Plot.RT.Aggregated.DataSetList <-
  function(dsList,
           aggr_on = 'funcId',
           targets = NULL,
           plot_mode = 'radar',
           use_rank = F,
           scale.ylog = T,
           maximize = T,
           erts = NULL,
           inf.action = 'overlap') {
    targets <- get_target_dt(dsList)
    data <-
      generate_data.Aggr(dsList, aggr_on = aggr_on, targets = targets)
    y_attr <- if (use_rank)
      'rank'
    else
      'value'
    y_title <- if (use_rank)
      'Rank'
    else
      'ERT'
    plot_general_data(
      data,
      type = plot_mode,
      x_attr = 'funcId',
      y_attr = y_attr,
      x_title = "FuncId",
      y_title = y_title,
      show.legend = T,
      scale.ylog = scale.ylog,
      inf.action = inf.action
    )
  }

#' @rdname Plot.FV.Aggregated
#' @export
Plot.FV.Aggregated.DataSetList <-
  function(dsList,
           aggr_on = 'funcId',
           runtimes = NULL,
           plot_mode = 'radar',
           use_rank = F,
           scale.ylog = T,
           fvs = NULL) {
    targets <- get_target_dt(dsList, which = 'by_FV')
    data <-
      generate_data.Aggr(dsList,
                         aggr_on = aggr_on,
                         targets = targets,
                         which = 'by_FV')
    y_attr <- if (use_rank)
      'rank'
    else
      'value'
    y_title <- if (use_rank)
      'Rank'
    else
      'Best-so-far f(x)'
    plot_general_data(
      data,
      type = plot_mode,
      x_attr = 'funcId',
      y_attr = y_attr,
      x_title = "FuncId",
      y_title = y_title,
      show.legend = T,
      scale.ylog = scale.ylog
    )
  }

#' @rdname Plot.Stats.Significance_Heatmap
#' @export
Plot.Stats.Significance_Heatmap.DataSetList <-
  function(dsList,
           ftarget,
           alpha = 0.01,
           bootstrap.size = 30,
           which = 'by_FV') {
    if (length(get_dim(dsList)) != 1 ||
        length(get_funcId(dsList)) != 1 ||
        length(get_id(dsList)) < 2)
      return(NULL)

    p_matrix <-
      pairwise.test(dsList, ftarget, bootstrap.size, which)
    y <- p_matrix <= alpha
    colorScale <- data.frame(
      x = c(-1, -0.33, -0.33, 0.33, 0.33, 1),
      col = c('blue', 'blue', 'white', 'white', 'red', 'red')
    )
    heatmap <-  y - t(y)
    heatmap[is.na(heatmap)] <- 0
    p <-
      plot_ly(
        x = colnames(y),
        y = rownames(y),
        z = heatmap,
        type = 'heatmap',
        xgap = 0.2,
        ygap = 0.2,
        colorscale = colorScale,
        showscale = F
      )
    p %<>% layout(
      yaxis = list(autorange = 'reversed', scaleratio = 1),
      xaxis = list(tickangle = 45)
    )
    p
  }

#' @rdname Plot.Comparison.Heatmap
#' @export
Plot.Comparison.Heatmap.DataSetList <-
  function(dsList,
           target_dt = NULL,
           which = 'by_FV') {
    matr <- generate_data.Heatmaps(dsList, which, target_dt)
    order <- rowMeans(matr) %>% sort(decreasing = T) %>% names
    matr <- matr[order[1:min(6, length(order))], order]
    p <-
      plot_ly(
        x = colnames(matr),
        y = rownames(matr),
        z = matr,
        type = 'heatmap',
        xgap = 0.2,
        ygap = 0.2,
        colorscale = 'RdBu',
        showscale = F,
        zmin = 0,
        zmax = 1
      )
    p %<>% layout(
      yaxis = list(autorange = 'reversed', scaleratio = 1),
      xaxis = list(tickangle = 45)
    )
    p
  }

#' Helper function for Plot.Stats.Significance_Graph
#'
#' @param x x
#' @param start default is 0
#' @param direction default is 1
#'
#' @noRd
radian.rescale <- function(x,
                           start = 0,
                           direction = 1) {
  c.rotate <- function(x)
    (x + start) %% (2 * pi) * direction
  c.rotate((2 * pi * (x - min(x)) / (max(x) - min(x))))
}

#' @rdname Plot.Stats.Significance_Graph
#' @export
Plot.Stats.Significance_Graph.DataSetList <-
  function(dsList,
           ftarget,
           alpha = 0.01,
           bootstrap.size = 30,
           which = 'by_FV') {
    if (!requireNamespace("igraph", quietly = TRUE)) {
      stop("Package \"igraph\" needed for this function to work. Please install it.",
           call. = FALSE)
    }
    if (length(get_dim(dsList)) != 1 ||
        length(get_funcId(dsList)) != 1 ||
        length(get_id(dsList)) < 2) {
      return(NULL)
    }
    p_matrix <-
      pairwise.test(dsList, ftarget, bootstrap.size, which)
    g <-
      igraph::graph_from_adjacency_matrix(p_matrix <= alpha, mode = 'directed', diag = F)
    lab.locs <-
      radian.rescale(x = 1:nrow(p_matrix),
                     direction = -1,
                     start = 0)

    igraph::plot.igraph(
      g,
      layout = igraph::layout.circle(g),
      vertex.size = 10,
      edge.arrow.size = 1,
      vertex.label.color = 'black',
      vertex.label.dist = 2,
      vertex.label.cex = 1,
      vertex.label.degree = lab.locs
    )
  }

#' @rdname Plot.Stats.Glicko2_Candlestick
#' @export
Plot.Stats.Glicko2_Candlestick.DataSetList <-
  function(dsList,
           nr_rounds = 100,
           glicko2_rank_df = NULL,
           which = 'by_FV',
           target_dt = NULL) {
    df <- glicko2_rank_df

    if (is.null(df)) {
      df <-
        glicko2_ranking(dsList, nr_rounds, which, target_dt = target_dt)$ratings
      Ids <- df$Player$ID
    }
    else{
      Ids <- df$ID
    }
    p <- IOH_plot_ly_default(title = "Glicko2-rating",
                             x.title = "ID",
                             y.title = "Rating")
    df$Rating %<>% as.numeric
    df$Deviation %<>% as.numeric
    high <- df$Rating + 3 * df$Deviation
    low <- df$Rating - 3 * df$Deviation
    open <- df$Rating + df$Deviation
    close <- df$Rating - df$Deviation

    N <- length(df$Rating)
    colors <- get_color_scheme(Ids)
    if (length(colors != N)) {
      colors <- get_color_scheme(get_id(dsList))
    }

    for (i in seq(N)) {
      # rgba_str <- paste0('rgba(', paste0(col2rgb(colors[i]), collapse = ','), ',0.52)')
      color <- list(line = list(color = colors[[i]]))
      p %<>% add_trace(
        type = "candlestick",
        x = Ids[[i]],
        open = open[[i]],
        close = close[[i]],
        high = high[[i]],
        low = low[[i]],
        legendgroup = Ids[[i]],
        name = Ids[[i]],
        increasing = color,
        decreasing = color,
        hovertext = paste0(
          format(df$Rating[[i]], digits = 3),
          '+-',
          format(df$Deviation[[i]], digits = 3)
        ),
        hoverinfo = "text"
      )
    }
    p %<>% layout(xaxis = list(rangeslider = list(visible = F)))
    p
  }

### _______________________ Rewritten plotting function ____________________ ###

#' Add transparancy to named list of colors
#'
#' @param colors Named list of colors (in hex-notation)
#' @param percentage The percentage of opacity. 0 is fully transparant, 1 is fully opaque
#'
#' @noRd
add_transparancy <- function(colors, percentage) {
  hex_val <-
    format(as.hexmode(as.integer(255 * percentage)),
           upper.case = T,
           width = 2)
  sapply(colors, function(col) {
    col <- paste0('#',  substr(col, 2, 7), hex_val)
  })
}

#' General function for plotting within IOHanalyzer
#'
#' @param df The dataframe containing the data to plot. It should contain at least two columns:
#' 'x_attr' and 'y_attr'
#' @param x_attr The column to specify the x_axis. Default is 'algId'
#' @param legend_attr Default is 'algId' This is also used for the selection of colorschemes
#' @param y_attr The column to specify the y_axis
#' @param type The type of plot to use. Currently available: 'violin', 'line', 'radar',
#' 'bar', hist' and 'ribbon'
#' @param upper_attr When using ribbon-plot, this can be used to create a shaded area.
#' Only works in combination with`lower_attr` and `type` == 'ribbon'
#' @param lower_attr When using ribbon-plot, this can be used to create a shaded area.
#' Only works in combination with`upper_attr` and `type` == 'ribbon'
#' @param subplot_attr Which attribute of the dataframe to use for creating subplots
#' @param subplot_shareX Whether or not to share X-axis when using subplots
#' @param scale.xlog Logarithmic scaling of x-axis
#' @param scale.ylog Logarithmic scaling of y-axis
#' @param scale.reverse Decreasing or increasing x-axis
#' @param x_title Title of x-axis. Defaults to x_attr
#' @param y_title Title of x-axis. Defaults to x_attr
#' @param plot_title Title of x-axis. Defaults to no title
#' @param p A previously existing plot on which to add traces. If NULL, a new canvas is created
#' @param show.legend Whether or not to include a legend
#' @param inf.action How to deal with infinite values. Can be 'none', 'overlap' or 'jitter'
#' @param violin.showpoints Wheteher or not to show individual points when making a violinplot
#' @param frame_attr Which attribute of the dataframe to use for the time element of the animation
#' @param symbol_attr Which attribute of the dataframe to use for the scatter symbol
#' @param line.step Whether to plot lines as a step-function (T) or as linear interpolation (F, default)
#' @param ... Additional parameters for the add_trace function
#'
#'
#' @export
plot_general_data <-
  function(df,
           x_attr = 'ID',
           y_attr = 'vals',
           type = 'violin',
           legend_attr = 'ID',
           scale.xlog = F,
           scale.ylog = F,
           scale.reverse = F,
           p = NULL,
           x_title = NULL,
           y_title = NULL,
           plot_title = NULL,
           upper_attr = NULL,
           lower_attr = NULL,
           subplot_attr = NULL,
           show.legend = F,
           inf.action = 'none',
           violin.showpoints = F,
           frame_attr = 'frame',
           symbol_attr = 'run_nr',
           subplot_shareX = F,
           line.step = F,
           ...) {
    l <-
      x <-
      isinf <-
      y <-
      text <-
      l_orig <- frame <- NULL #Set local binding to remove warnings

    #Only allow valid plot types
    if (!(
      type %in% c(
        'violin',
        'line',
        'radar',
        'hist',
        'ribbon',
        'line+ribbon',
        'bar',
        'anim-scatter',
        'scatter'
      )
    )) {
      stop(paste0("Provided plot type ('", type, "') is not supported"))
    }

    #And valid number of y-attributes
    if (length(y_attr) == 0) {
      stop("At least one y-attribute is needed to plot")
    }

    #Deal with subplots
    if (!is.null(subplot_attr)) {
      if (!subplot_attr %in% colnames(df)) {
        stop("Provided subplot-attribut is not a colname of the selected data.table.")
      }
      colnames(df)[colnames(df) == subplot_attr] <- "subplot_attr"
      attrs <- unique(df[, subplot_attr])
      if (length(attrs) == 0)
        stop(
          "Attempting to create subplots with fewer than 2 unique values of
                                 `subplot_attrs`-column"
        )
      if (length(attrs) == 1)
        return(
          plot_general_data(
            df,
            x_attr,
            y_attr,
            type,
            legend_attr,
            scale.xlog,
            scale.ylog,
            scale.reverse,
            p,
            x_title,
            y_title,
            attrs,
            upper_attr,
            lower_attr,
            show.legend = show.legend,
            subplot_attr = NULL,
            ...
          )
        )
      if (subplot_attr == legend_attr) {
        df[, l := subplot_attr]
      }

      #Only need one legend for the whole plot
      legends_show <- rep(F, length(attrs))
      legends_show[[1]] <- show.legend
      names(legends_show) <- as.character(attrs)

      #Get some number of rows and columns
      n_cols <- 1 + ceiling(length(attrs) / 10)
      n_rows <- ceiling(length(attrs) / n_cols)

      p <- lapply(seq(length(attrs)), function(idx) {
        attr_val <- attrs[[idx]]
        df_sub <- df[subplot_attr == attr_val]
        disp_y <-  idx %% n_cols == 1
        disp_x <- idx > (length(attrs) - n_cols)
        x.title = if (disp_x)
          x_title
        else
          ""
        y.title = if (disp_y)
          y_title
        else
          ""

        #Generate title for the subplots
        if (stri_detect_regex(subplot_attr, "(?i)fun"))
          sub_title <- paste0('F', attr_val)
        else if (stri_detect_regex(subplot_attr, "(?i)dim"))
          sub_title <- paste0('D', attr_val)
        else
          sub_title <- paste0(attr_val)
        p <- NULL
        if (stri_detect_fixed(type, '+')) {
          type1 <-
            substr(type, 0, stri_locate_all(type, fixed = '+')[[1]][[1]] - 1)
          p <-
            plot_general_data(
              df_sub,
              x_attr,
              y_attr,
              type1,
              legend_attr,
              scale.xlog,
              scale.ylog,
              scale.reverse,
              NULL,
              x.title,
              y.title,
              plot_title,
              upper_attr,
              lower_attr,
              show.legend = legends_show[[as.character(attr_val)]],
              subplot_attr = NULL,
              ...
            )
          type <-
            substr(type,
                   stri_locate_all(type, fixed = '+')[[1]][[1]] + 1,
                   nchar(type))
        }
        p <-
          plot_general_data(
            df_sub,
            x_attr,
            y_attr,
            type,
            legend_attr,
            scale.xlog,
            scale.ylog,
            scale.reverse,
            p,
            x.title,
            y.title,
            plot_title,
            upper_attr,
            lower_attr,
            show.legend = legends_show[[as.character(attr_val)]],
            subplot_attr = NULL,
            ...
          )
        if (getOption("IOHanalyzer.annotation_x", 0.5) >= 0 &
            getOption("IOHanalyzer.annotation_y", 1) >= 0) {
          p %<>% layout(
            annotations = list(
              text = sub_title,
              font = f2,
              showarrow = FALSE,
              xref = "paper",
              yref = "paper",
              x = getOption("IOHanalyzer.annotation_x", 0.5),
              y = getOption("IOHanalyzer.annotation_y", 1)
            )
          )
          p
        }

      })

      p <- subplot(
        p,
        nrows = n_rows,
        titleX = T,
        titleY = T,
        margin = c(
          getOption("IOHanalyzer.margin_horizontal", 0.02),
          getOption("IOHanalyzer.margin_vertical", 0.02),
          getOption("IOHanalyzer.margin_horizontal", 0.02),
          getOption("IOHanalyzer.margin_vertical", 0.02)
        ),
        shareX = subplot_shareX
      ) %>%
        layout(title = plot_title)
      return(p)
    }

    # Replace colnames to have easier matching
    if (!x_attr %in% colnames(df) ||
        !all(y_attr %in% colnames(df))) {
      stop("Not all provided attributes are colnames of the selected data.table.")
    }
    colnames(df)[colnames(df) == x_attr] <- "x"


    if (length(y_attr) == 1 && type != 'line')
      colnames(df)[colnames(df) == y_attr] <- "y"
    else if (type != 'line')
      stop("Multiple y-attrs is currently only supported for line-plots")

    if (!is.null(upper_attr) && !is.null(lower_attr)) {
      if (!upper_attr %in% colnames(df) ||
          !lower_attr %in% colnames(df)) {
        stop("Provided upper and lower attributes are not colnames of the selected data.table.")
      }
      colnames(df)[colnames(df) == upper_attr] <- "upper"
      colnames(df)[colnames(df) == lower_attr] <- "lower"
    }

    if (x_attr != legend_attr) {
      colnames(df)[colnames(df) == legend_attr] <- "l"
      xs <- unique(df[['l']])
    }
    else{
      xs <- unique(df[['x']])
    }

    #Get color and based on legend-attribute
    colors <- get_color_scheme(xs)
    if (is.null(names(colors)) ||
        !all(names(colors) %in% xs))
      names(colors) <- xs

    xscale <- if (scale.xlog)
      'log'
    else
      'linear'
    yscale <- if (scale.ylog)
      'log'
    else
      'linear'

    #If new plot is needed, create one. Store in bool to decide if axis scaling is needed.
    is_new_plot <- F
    if (is.null(p)) {
      p <-
        IOH_plot_ly_default(
          x.title = ifelse(is.null(x_title), x_attr, x_title),
          y.title = ifelse(is.null(y_title), y_attr, y_title),
          title = plot_title
        )
      is_new_plot <- T
    }

    switch(
      type,
      'violin' = {
        if (legend_attr != x_attr) {
          warning("Inconsistent attribute selected for x-axis and legend. Using x_attr as name")
        }
        #Update names to aviod numerical legend
        if (is.numeric(df[['x']])) {
          if (stri_detect_regex(x_attr, "(?i)fun"))
            df <- df[, x := paste0('F', sprintf("%02d", x))]
          else if (stri_detect_regex(x_attr, "(?i)dim"))
            df <- df[, x := paste0('D', as.character(x))]
          else
            df <- df[, x := as.character(x)]
        }
        #Update color names as well, since the value changed
        names(colors) <- unique(df[['x']])

        p %<>%
          add_trace(
            data = df,
            x = ~ x,
            y = ~ y,
            type = 'violin',
            hoveron = "points+kde",
            points = violin.showpoints,
            pointpos = 1.5,
            jitter = 0,
            scalemode = 'count',
            meanline = list(visible = F),
            name = ~ x,
            colors = colors,
            color = ~ x,
            split = ~ x,
            line = list(color = 'black', width = 1.1),
            box = list(visible = T),
            spanmode = 'hard',
            showlegend = show.legend,
            ...
          )
        if (is_new_plot) {
          p %<>% layout(yaxis = list(
            type = yscale,
            tickfont = f3(),
            ticklen = 3
          ))
        }
      },
      'line' = {
        if (legend_attr == x_attr) {
          stop("Duplicated attribute selected for x-axis and legend.")
        }

        # Force legend to be categorical
        df[, l_orig := l]
        if (is.numeric(df[['l']])) {
          df[, l := paste0('A', l)]
          names(colors) <- paste0('A', names(colors))
        }

        #Use linestyles to differentiate traces if only one attribute is selected to be plotted
        #TODO: Combine these two options more elegantly
        if (length(y_attr) == 1) {
          dashes <- get_line_style(xs)
          names(dashes) <- xs
          colnames(df)[colnames(df) == y_attr] <- "y"

          df[, isinf := is.infinite(y)]
          df[, text := as.character(round(y, getOption("IOHanalyzer.precision", 2)))]

          if (inf.action == 'overlap') {
            maxval <- max(df[isinf == F, 'y'])
            df[['y']][df[['isinf']]] <-
              10 ** (ceiling(log10(maxval)) + 1)
          }
          else if (inf.action == 'jitter') {
            #TODO: Faster way to compute this
            maxval <- max(df[isinf == F, 'y'])
            for (xval in unique(df[['x']])) {
              tempval <- 10 ** (ceiling(log10(maxval)) + 1)
              for (lval in unique(df[['l']])) {
                temp <- df[l == lval][x == xval]
                if (nrow(temp) > 0 &&
                    df[l == lval][x == xval][['isinf']]) {
                  df[l == lval][x == xval][['y']] <- tempval
                  tempval <- 1.2 * tempval
                }
              }
            }
          }

          suppressWarnings(
            p %<>%
              add_trace(
                data = df,
                x = ~ x,
                y = ~ y,
                color = ~ l,
                legendgroup = ~ l_orig,
                name = ~ l_orig,
                type = 'scatter',
                mode = 'lines+markers',
                linetype = ~ l_orig,
                marker = list(size = getOption('IOHanalyzer.markersize', 4)),
                linetypes = dashes,
                colors = colors,
                showlegend = show.legend,
                text = ~ text,
                line = list(
                  width = getOption('IOHanalyzer.linewidth', 2),
                  shape = ifelse(line.step, "hv", "linear")
                ),
                hovertemplate = '%{text}',
                ...
              )
          )
          if (inf.action != 'none') {
            p %<>% add_trace(
              data = df[isinf == T],
              x = ~ x,
              y = ~ y,
              legendgroup = ~ l_orig,
              name = ~ l_orig,
              type = 'scatter',
              mode = 'markers',
              color = ~ l,
              marker = list(
                symbol = 'circle-open',
                size = 8 + getOption('IOHanalyzer.markersize', 4)
              ),
              colors = colors,
              showlegend = F,
              text = 'Inf',
              hoverinfo = 'none',
              ...
            )
          }

        }
        else {
          if (inf.action != 'none') {
            warning("inf.action is not yet supported for multiple y-attributes")
          }

          dashes_full <-
            rep(
              c(
                "solid",
                "dot",
                "dash",
                "longdash",
                "dashdot",
                "longdashdot"
              ),
              ceiling(length(y_attr) / 3)
            )[1:length(y_attr)]
          names(dashes_full) <- y_attr

          for (y_atr in y_attr) {
            colnames(df)[colnames(df) == y_atr] <- "y"

            #TODO: Figure out how to supress warning about 6 linetypes
            dashstyle <- dashes_full[[y_atr]]
            suppressWarnings(
              p %<>%
                add_trace(
                  data = df,
                  x = ~ x,
                  y = ~ y,
                  color = ~ l,
                  legendgroup = ~ l_orig,
                  name = ~ l_orig,
                  type = 'scatter',
                  mode = 'lines+markers',
                  marker = list(size = getOption('IOHanalyzer.markersize', 4)),
                  linetype = dashstyle,
                  colors = colors,
                  showlegend = show.legend,
                  name = ~ l,
                  text = y_atr,
                  line = list(
                    width = getOption('IOHanalyzer.linewidth', 2),
                    shape = ifelse(line.step, "hv", "linear")
                  ),
                  ...
                )
            )
            colnames(df)[colnames(df) == "y"] <- y_atr
            show.legend <- F
          }
        }
        if (is_new_plot) {
          if (is.numeric(df[['x']]))
            p %<>% layout(
              xaxis = list(
                type = xscale,
                tickfont = f3(),
                ticklen = 3,
                autorange = ifelse(scale.reverse, "reversed", T)
              ),
              yaxis = list(
                type = yscale,
                tickfont = f3(),
                ticklen = 3
              )
            )
          else
            p %<>% layout(
              xaxis = list(
                type = 'category',
                tickfont = f3(),
                ticklen = 3
              ),
              yaxis = list(
                type = yscale,
                tickfont = f3(),
                ticklen = 3
              )
            )

        }
      },
      'ribbon' = {
        if (legend_attr == x_attr) {
          stop("Duplicated attribute selected for x-axis and legend.")
        }
        if (is.null(upper_attr) || is.null(lower_attr)) {
          stop("No upper or lower attribute provided for ribbon-plot")
        }

        for (name in xs) {
          df_small <- df[l == name]
          legend_name <- as.character(name)
          rgba_str <-
            paste0('rgba(', paste0(col2rgb(colors[[name]]), collapse = ','), ',0.2)')
          p %<>%
            add_trace(
              data = df_small,
              x = ~ x,
              y = ~ upper,
              type = 'scatter',
              mode = 'lines',
              line = list(color = rgba_str, width = 0),
              legendgroup = legend_name,
              showlegend = F,
              name = 'upper',
              ...
            ) %>%
            add_trace(
              x = ~ x,
              y = ~ lower,
              type = 'scatter',
              mode = 'lines',
              fill = 'tonexty',
              line = list(color = 'transparent'),
              legendgroup = legend_name,
              fillcolor = rgba_str,
              showlegend = F,
              name = 'lower',
              ...
            )
        }



        if (is_new_plot) {
          p %<>% layout(
            xaxis = list(
              type = xscale,
              tickfont = f3(),
              ticklen = 3,
              autorange = ifelse(scale.reverse, "reversed", T)
            ),
            yaxis = list(
              type = yscale,
              tickfont = f3(),
              ticklen = 3
            )
          )
        }
      },
      'radar' = {
        if (legend_attr == x_attr) {
          stop("Duplicated attribute selected for x-axis and legend.")
        }
        if (is.numeric(df[['x']])) {
          if (stri_detect_regex(x_attr, "(?i)fun"))
            df <- df[, x := paste0('F', sprintf("%02d", x))]
          else if (stri_detect_regex(x_attr, "(?i)dim"))
            df <- df[, x := paste0('D', as.character(x))]
          else
            df <- df[, x := as.character(x)]
        }
        df <- df[, col := add_transparancy(colors, 0.4)[l]]
        p %<>%
          add_trace(
            data = df,
            type = 'scatterpolar',
            r = ~ y,
            theta = ~ x,
            mode = 'markers',
            #marker = list(color = 'lightgrey', size=0),
            fill = 'toself',
            connectgaps = T,
            fillcolor = ~ col,
            color = ~ l,
            colors = colors,
            name =  ~ l,
            legendgroup = ~ l,
            ...
          )
        if (is_new_plot) {
          p %<>% layout(polar = list(
            radialaxis = list(
              type = yscale,
              tickfont = f3(),
              ticklen = 3,
              autorange = ifelse(scale.reverse, "reversed", T)
            )
          ))
        }
      },
      'hist' = {
        if (legend_attr == x_attr) {
          stop("Duplicated attribute selected for x-axis and legend.")
        }
        if (!'width' %in% colnames(df)) {
          stop(
            "No 'width'-column included in the provided dataframe. This is required for a histogram-plot"
          )
        }
        p %<>%
          add_trace(
            data = df,
            x = ~ x,
            y = ~ y,
            width = ~ width,
            type = 'bar',
            name = ~ l,
            text = ~ text,
            hoverinfo = 'text',
            colors = add_transparancy(colors, 0.6),
            color = ~ l,
            marker = list(line = list(color = 'rgb(8,48,107)')),
            ...
          )

        if (is_new_plot) {
          p %<>% layout(
            xaxis = list(
              type = xscale,
              tickfont = f3(),
              ticklen = 3,
              autorange = ifelse(scale.reverse, "reversed", T)
            ),
            yaxis = list(
              type = yscale,
              tickfont = f3(),
              ticklen = 3
            )
          )
        }
      },
      'bar' = {
        if (legend_attr != x_attr) {
          warning("Inconsistent attribute selected for x-axis and legend. Using x_attr as name")
        }
        colors = add_transparancy(colors, 0.6)
        for (xv in xs) {
          p %<>%
            add_trace(
              x = xv,
              y = df[x == xv, y],
              type = 'bar',
              name = xv,
              color = colors[xv],
              marker = list(line = list(color = 'rgb(8,48,107)')),
              ...
            )
        }

        if (is_new_plot) {
          p %<>% layout(
            xaxis = list(tickfont = f3(), ticklen = 3),
            yaxis = list(
              type = yscale,
              tickfont = f3(),
              ticklen = 3
            )
          )
        }
      },
      'anim-scatter' = {
        colnames(df)[colnames(df) == frame_attr] <- "frame"
        colnames(df)[colnames(df) == symbol_attr] <- "s"
        colors = add_transparancy(colors, 0.9)
        df = df[order(frame), ]
        for (xv in xs) {
          df_sub = df[l == xv,]
          p %<>% add_trace(
            data = df_sub,
            x = ~ x,
            y = ~ y,
            type = 'scatter',
            mode = 'markers',
            marker = list(color = colors[xv],
                          symbol = ~
                            s),
            legendgroup = xv,
            frame = ~ frame,
            showlegend = F,
            name = xv
          )
        }
        p %<>% animation_opts(transition = 0)

      },
      'scatter' = {
        colnames(df)[colnames(df) == symbol_attr] <- "s"
        colors = add_transparancy(colors, 0.9)
        df = df[order(frame), ]
        for (xv in xs) {
          df_sub = df[l == xv,]
          p %<>% add_trace(
            data = df_sub,
            x = ~ x,
            y = ~ y,
            type = 'scatter',
            mode = 'markers',
            marker = list(color = colors[xv],
                          symbol = ~
                            s),
            legendgroup = xv,
            showlegend = F,
            name = xv
          )
        }
      }
    )
    return(p)
  }


#' Create the PerformViz plot
#'
#' From the paper:
#'
#' @param DSC_rank_result The result from a call to DSCtool rank service (`get_dsc_rank`)
#'
#' @return A performviz plot
#' @export
#' @examples
#' \dontrun{
#' Plot.Performviz(get_dsc_rank(dsl))
#' }
Plot.Performviz <- function(DSC_rank_result) {
  if (!requireNamespace("ComplexHeatmap", quietly = TRUE)) {
    stop(
      "Package \"ComplexHeatmap\" needed for this function to work. Please install it.",
      call. = FALSE
    )
  }
  if (!requireNamespace("reshape2", quietly = TRUE)) {
    stop("Package \"reshape2\" needed for this function to work. Please install it.",
         call. = FALSE)
  }
  if (!requireNamespace("grid", quietly = TRUE)) {
    stop("Package \"grid\" needed for this function to work. Please install it.",
         call. = FALSE)
  }
  mlist <- DSC_rank_result$ranked_matrix

  problem <- NULL #Assign variable to remove warnings
  # df_temp <- rbindlist(lapply(mlist[[problem_idx]]$result,
  #                             function(x) {
  #                               list(algorithm = x$algorithm, rank =  x$rank)
  #                             }))
  # df_temp[, problem := mlist[[problem_idx]]$problem]

  df <- rbindlist(lapply(seq(length(mlist)), function(problem_idx) {
    df_temp <- rbindlist(lapply(mlist[[problem_idx]]$result,
                                function(x) {
                                  list(algorithm = x$algorithm, rank =  x$rank)
                                }))
    df_temp[, problem := mlist[[problem_idx]]$problem]
  }))

  rank_matrix <-
    reshape2::acast(df, algorithm ~ problem, value.var = 'rank')
  df <- rank_matrix
  # colnames(df)<-index
  # rownames(df)<-vector
  # Define some graphics to display the distribution of columns
  # library(ComplexHeatmap)
  .hist = ComplexHeatmap::anno_histogram(df, gp = grid::gpar(fill = "lightblue"))
  .density = ComplexHeatmap::anno_density(df, type = "line", gp = grid::gpar(col = "blue"))
  ha_mix_top = ComplexHeatmap::HeatmapAnnotation(hist = .hist, density = .density)
  # Define some graphics to display the distribution of rows
  .violin = ComplexHeatmap::anno_density(
    df,
    type = "violin",
    gp = grid::gpar(fill = "lightblue"),
    which = "row"
  )
  .boxplot = ComplexHeatmap::anno_boxplot(df, which = "row")
  ha_mix_right = ComplexHeatmap::HeatmapAnnotation(
    violin = .violin,
    bxplt = .boxplot,
    which = "row",
    width = grid::unit(4, "cm")
  )
  # Combine annotation with heatmap
  heatmap_main <- ComplexHeatmap::Heatmap(
    df,
    name = "Ranking",
    column_names_gp = grid::gpar(fontsize = 8),
    top_annotation = ha_mix_top,
    top_annotation_height = grid::unit(3.8, "cm")
  )
  return(ComplexHeatmap::draw(
    ComplexHeatmap::`+.AdditiveUnit`(heatmap_main, ha_mix_right)
  ))
}

#' Plot the cumulative difference plot given a DataSetList.
#'
#' @param dsList A DataSetList (should consist of only one function and dimension and two algorithms).
#' @param runtime_or_target_value The target runtime or the target value
#' @param isFixedBudget Should be TRUE when target runtime is used. False otherwise.
#' @param alpha 1 minus the confidence level of the confidence band.
#' @param EPSILON If abs(x-y) < EPSILON, then we assume that x = y.
#' @param nOfBootstrapSamples The number of bootstrap samples used in the estimation.
#' @param dataAlreadyComputed If false, `generate_data.CDP` will be called to process the data.
#' @param precomputedData only needed when dataAlreadyComputed=TRUE. The result of `generate_data.CDP`.
#' @return A cumulative difference plot.
#' @export
#' @examples
#' dsl
#' dsl_sub <- subset(dsl, funcId == 1)
#' target <- 15
#'
#' Plot.cumulative_difference_plot(dsl_sub, target, FALSE)
Plot.cumulative_difference_plot <-
  function(dsList,
           runtime_or_target_value,
           isFixedBudget,
           alpha = 0.05,
           EPSILON = 1e-80,
           nOfBootstrapSamples = 1e3,
           dataAlreadyComputed = FALSE,
           precomputedData = NULL)
  {
    if (!requireNamespace("RVCompare", quietly = TRUE)) {
      stop("Package \"RVCompare\" needed for this function to work. Please install it.",
           call. = FALSE)
    }

    if (dataAlreadyComputed)
    {
      if (is.null(precomputedData))
      {
        return(NULL)
      }
      data <- precomputedData
    }
    else
    {
      data <-
        generate_data.CDP(
          dsList,
          runtime_or_target_value,
          isFixedBudget,
          alpha,
          EPSILON,
          nOfBootstrapSamples
        )
    }

    if (isFixedBudget)
    {
      subds <-
        get_FV_sample(dsList, runtime_or_target_value, output = 'long')

      algorithms <- unique(subds$ID)
    }
    else
    {
      subds <-
        get_RT_sample(dsList, runtime_or_target_value, output = 'long')

      algorithms <- unique(subds$ID)
    }





    # Convert back to list
    plot_data <- list()
    for (i in 1:ncol(data)) {
      plot_data[[i]] <- data[, i]
    }
    names(plot_data) <- colnames(data)


    # Confidence band
    trace1 <-
      list(
        x = plot_data$p,
        y = plot_data$diff_upper,
        line = list(color = "rgba(0, 0, 40, 0)"),
        mode = "lines",
        name = "Upper bound of the confidence band",
        type = "scatter"
      )
    trace3 <-
      list(
        x = plot_data$p,
        y = plot_data$diff_lower,
        connectgaps = TRUE,
        line = list(color = "rgba(0, 0, 40, 0)"),
        mode = "lines",
        name = "Lower bound of the confidence band",
        type = "scatter"
      )

    # Area in which the cumulative diference can be.
    trace4 <-
      list(
        x = c(0, 0.5, 1),
        y = c(0, 1, 0),
        line = list(color = "rgba(0, 0, 40, 0)"),
        mode = "lines",
        name = "",
        type = "scatter"
      )
    trace6 <- list(
      x = c(0, 0.5, 1),
      y = c(0, -1, 0),
      connectgaps = TRUE,
      line = list(color = "rgba(0, 0, 40, 0)"),
      mode = "lines",
      name = "",
      type = "scatter"
    )




    fig <- plotly::plot_ly() %>%
      plotly::add_lines(
        x = plot_data$p,
        y = plot_data$diff_estimation,
        color = I("black"),
        name = "Estimated cumulative diference"
      ) %>%
      plotly::add_lines(
        x = trace1$x,
        y = trace1$y,
        line = trace1$line,
        mode = trace1$mode,
        name = trace1$name,
        type = trace1$type,
        uid = trace1$uid,
        xsrc = trace1$xsrc,
        ysrc = trace1$ysrc
      ) %>%

      plotly::add_trace(
        x = trace3$x,
        y = trace3$y,
        connectgaps = trace3$connectgaps,
        line = trace3$line,
        mode = trace3$mode,
        name = trace3$name,
        type = trace3$type,
        uid = trace3$uid,
        xsrc = trace3$xsrc,
        ysrc = trace3$ysrc,
        fillcolor = "rgba(0,40,100,0.2)",
        fill = 'tonexty'
      ) %>%

      plotly::add_lines(
        x = trace4$x,
        y = trace4$y,
        line = trace4$line,
        mode = trace4$mode,
        name = trace4$name,
        type = trace4$type,
        uid = trace4$uid,
        xsrc = trace4$xsrc,
        ysrc = trace4$ysrc
      ) %>%
      plotly::add_trace(
        x = trace6$x,
        y = trace6$y,
        connectgaps = trace6$connectgaps,
        line = trace6$line,
        mode = trace6$mode,
        name = trace6$name,
        type = trace6$type,
        uid = trace6$uid,
        xsrc = trace6$xsrc,
        ysrc = trace6$ysrc,
        fillcolor = "rgba(51,204,255,0.2)",
        fill = 'tonexty'
      ) %>%


      plotly::layout(
        xaxis = list(range = c(0, 1)),
        yaxis = list(range = c(-1, 1)),
        showlegend = FALSE
      ) %>%

      plotly::add_annotations(
        x = 0.02,
        y = 0.85,
        xref = "x",
        yref = "y",
        text = algorithms[1],
        xanchor = 'left',
        showarrow = F
      ) %>%
      plotly::add_annotations(
        x = 0.02,
        y = -0.85,
        xref = "x",
        yref = "y",
        text = algorithms[2],
        xanchor = 'left',
        showarrow = F
      )

    return(fig)
  }


#' Create EAF-based polygon plots
#'
#'
#'
#' @param df The dataframe containing the data to plot. This should come from `generate_data.EAF`
#' @param subplot_attr Which attribute of the dataframe to use for creating subplots
#' @param subplot_shareX Whether or not to share X-axis when using subplots
#' @param scale.xlog Logarithmic scaling of x-axis
#' @param scale.ylog Logarithmic scaling of y-axis
#' @param xmin Minimum value for the x-axis
#' @param xmax Maximum value for the x-axis
#' @param ymin Minimum value for the y-axis
#' @param ymax Maximum value for the y-axis
#' @param maximization Whether the data comes from maximization or minimization
#' @param scale.reverse Decreasing or increasing x-axis
#' @param x_title Title of x-axis. Defaults to x_attr
#' @param y_title Title of x-axis. Defaults to x_attr
#' @param plot_title Title of x-axis. Defaults to no title
#' @param p A previously existing plot on which to add traces. If NULL, a new canvas is created
#' @param show.colorbar Whether or not to include a colorbar
#' @param dt_overlay Dataframe containing additional data (e.g. quantiles) to plot
#' on top of the EAF. This should have a column labeled 'runtime'. The other columsn will
#' all be plotted as function values.
#' @param ... Additional parameters for the add_trace function
#'
#' @return An EAF plot
#' @export
#' @examples
#' \dontrun{
#' plot_eaf_data(generate_data.EAF(subset(dsl, ID==get_id(dsl)[[1]])), maximization=T)
#' }
plot_eaf_data <-
  function(df,
           maximization = F,
           scale.xlog = F,
           scale.ylog = F,
           scale.reverse = F,
           p = NULL,
           x_title = NULL,
           xmin = NULL,
           xmax = NULL,
           ymin = NULL,
           ymax = NULL,
           y_title = NULL,
           plot_title = NULL,
           subplot_attr = NULL,
           show.colorbar = F,
           subplot_shareX = F,
           dt_overlay = NULL,
           ...) {
    l <-
      x <-
      isinf <-
      y <-
      text <-
      l_orig <- frame <- NULL #Set local binding to remove warnings

    #Deal with subplots
    if (!is.null(subplot_attr)) {
      if (!subplot_attr %in% colnames(df)) {
        stop("Provided subplot-attribut is not a colname of the selected data.table.")
      }
      colnames(df)[colnames(df) == subplot_attr] <- "subplot_attr"
      if (!is.null(dt_overlay)) {
        colnames(dt_overlay)[colnames(dt_overlay) == subplot_attr] <-
          "subplot_attr"
      }
      attrs <- unique(df[, subplot_attr])
      if (length(attrs) == 0)
        stop(
          "Attempting to create subplots with fewer than 2 unique values of
                                 `subplot_attrs`-column"
        )
      if (length(attrs) == 1)
        return(
          plot_eaf_data(
            df,
            maximization = maximization,
            scale.xlog = scale.xlog,
            scale.ylog = scale.ylog,
            scale.reverse = scale.reverse,
            p = p,
            x_title = x_title,
            xmin = xmin,
            xmax = xmax,
            ymin = ymin,
            ymax = ymax,
            y_title = y_title,
            show.colorbar = show.colorbar,
            subplot_attr = NULL,
            dt_overlay = dt_overlay,
            ...
          )
        )

      #Get some number of rows and columns
      n_cols <- 1 + ceiling(length(attrs) / 10)
      n_rows <- ceiling(length(attrs) / n_cols)

      p <- lapply(seq(length(attrs)), function(idx) {
        attr_val <- attrs[[idx]]
        df_sub <- df[subplot_attr == attr_val]
        dt_overlay_sub <- dt_overlay[subplot_attr == attr_val]
        disp_y <-  idx %% n_cols == 1
        disp_x <- idx > (length(attrs) - n_cols)
        x.title = if (disp_x)
          x_title
        else
          ""
        y.title = if (disp_y)
          y_title
        else
          ""

        #Generate title for the subplots
        if (stri_detect_regex(subplot_attr, "(?i)fun"))
          sub_title <- paste0('F', attr_val)
        else if (stri_detect_regex(subplot_attr, "(?i)dim"))
          sub_title <- paste0('D', attr_val)
        else
          sub_title <- paste0(attr_val)
        p <- NULL
        p <-
          plot_eaf_data(
            df_sub,
            maximization = maximization,
            scale.xlog = scale.xlog,
            scale.ylog = scale.ylog,
            scale.reverse = scale.reverse,
            p = p,
            x_title = x_title,
            xmin = xmin,
            xmax = xmax,
            ymin = ymin,
            ymax = ymax,
            y_title = y_title,
            show.colorbar = F,
            subplot_attr = NULL,
            dt_overlay = dt_overlay_sub,
            ...
          )
        if (getOption("IOHanalyzer.annotation_x", 0.5) >= 0 &
            getOption("IOHanalyzer.annotation_y", 1) >= 0) {
          p %<>% layout(
            annotations = list(
              text = sub_title,
              font = f2,
              showarrow = FALSE,
              xref = "paper",
              yref = "paper",
              x = getOption("IOHanalyzer.annotation_x", 0.5),
              y = getOption("IOHanalyzer.annotation_y", 1)
            )
          )
          p
        }

      })

      p <- subplot(
        p,
        nrows = n_rows,
        titleX = T,
        titleY = T,
        margin = c(
          getOption("IOHanalyzer.margin_horizontal", 0.02),
          getOption("IOHanalyzer.margin_vertical", 0.02),
          getOption("IOHanalyzer.margin_horizontal", 0.02),
          getOption("IOHanalyzer.margin_vertical", 0.02)
        ),
        shareX = subplot_shareX
      ) %>%
        layout(title = plot_title)
      return(p)
    }

    xscale <- if (scale.xlog)
      'log'
    else
      'linear'
    yscale <- if (scale.ylog)
      'log'
    else
      'linear'




    #If new plot is needed, create one. Store in bool to decide if axis scaling is needed.
    is_new_plot <- F
    if (is.null(p)) {
      p <- IOH_plot_ly_default(x.title = x_title,
                               y.title = y_title,
                               title = plot_title)
      is_new_plot <- T
    }

    eaf_sets <- df$`percentage`
    uniq_eaf_sets <- unique(eaf_sets)
    att_surfs <- split.data.frame(df[, .(`runtime`, `f(x)`)],
                                  factor(eaf_sets,
                                         levels = uniq_eaf_sets,
                                         labels = uniq_eaf_sets))
    cols <- rev(viridis(length(att_surfs)))
    if (maximization)
      extreme = as.matrix(df[, .(runtime = max(`runtime`), `f(x)` = min(`f(x)`))])
    else
      extreme = as.matrix(df[, .(runtime = max(`runtime`), `f(x)` = max(`f(x)`))])

    for (i in seq_along(att_surfs)) {
      poli <-
        add.extremes(points.steps(as.matrix(att_surfs[[i]])),
                     as.matrix(extreme),
                     c(F, maximization))
      poli <- rbind(poli, extreme)

      p %<>% add_polygons(
        poli[, 'runtime'],
        poli[, 'f(x)'],
        alpha = 1,
        fillcolor = cols[i],
        line = list(width = 0),
        name = names(att_surfs)[i],
        showlegend = F
      )
    }

    # Set axis ranges
    xmin <-
      ifelse((is.null(xmin) ||
                xmin == ""), min(df$`runtime`), as.numeric(xmin))
    xmax <-
      ifelse((is.null(xmax) ||
                xmax == ""), max(df$`runtime`), as.numeric(xmax))
    if (scale.xlog) {
      xmin <- log10(xmin)
      xmax <- log10(xmax)
    }
    ymin <-
      ifelse((is.null(ymin) ||
                ymin == ""), min(df$`f(x)`), as.numeric(ymin))
    ymax <-
      ifelse((is.null(ymax) ||
                ymax == ""), max(df$`f(x)`), as.numeric(ymax))
    if (scale.ylog) {
      ymin <- log10(ymin)
      ymax <- log10(ymax)
    }
    yrange <- c(ymin, ymax)
    if (scale.reverse)
      yrange <- rev(yrange)

    p %<>% layout(
      xaxis = list(
        type = xscale,
        tickfont = f3(),
        ticklen = 3,
        range = c(xmin, xmax)
      ),
      yaxis = list(
        type = yscale,
        tickfont = f3(),
        ticklen = 3,
        range = yrange
      )
    )

    if (!is.null(dt_overlay)) {
      cnames <- colnames(dt_overlay)
      if (!('runtime' %in% cnames)) {
        warning('dt_overlay needs to contain a columns labeled `runtime` to be used.')
      } else {
        for (cname in cnames[!(cnames %in% c('runtime', 'subplot_attr', 'ID'))]) {
          p %<>% add_trace(
            x = dt_overlay[['runtime']],
            y = dt_overlay[[cname]],
            type = 'scatter',
            mode = 'lines',
            name = cname,
            showlegend = F,
            line = list(
              width = getOption('IOHanalyzer.linewidth', 2),
              color = 'black',
              shape = 'hv'
            )
          )
        }
      }
    }
    if (show.colorbar) {
      p %<>% add_contour(
        z = matrix(-0.1, 1.1),
        zmin = -0.1,
        zmax = 1.1,
        colorscale = 'Viridis',
        contours = list(coloring = 'fill'),
        reversescale = T
      )
      p %<>% colorbar(
        cmin = -0,
        cmax = 1,
        thickness = 0.03,
        thicknessmode = 'fraction',
        len = 1,
        tickvals = c(0, 0.5, 1),
        tickmode = 'array',
        outlinewidth = 1,
        title = 'Fraction'
      )
    }

    p

    return(p)
  }



#' Create EAF-difference contour plots
#'
#'
#'
#' @param matrices The dataframes containing the data to plot. This should come from `generate_data.EAF_diff_Approximate`
#' @param scale.xlog Logarithmic scaling of x-axis
#' @param scale.ylog Logarithmic scaling of y-axis
#' @param zero_transparant Whether values of 0 should be made transparant or not
#' @param show_negatives Whether to also show negative values or not
#'
#' @return EAF difference plots
#' @export
#' @examples
#' \dontrun{
#' plot_eaf_differences(generate_data.EAF_diff_Approximate(subset(dsl, funcId == 1), 1, 50, 1, 16))
#' }
plot_eaf_differences <-
  function(matrices,
           scale.xlog = T,
           scale.ylog = F,
           zero_transparant = F,
           show_negatives = F) {
    xscale <- if (scale.xlog)
      'log'
    else
      'linear'
    yscale <- if (scale.ylog)
      'log'
    else
      'linear'


    show_colorbar <- T
    ids <- names(matrices)
    ps <- lapply(seq(length(ids)), function(idx) {
      diff <- matrices[[idx]]
      if (!show_negatives)
        diff[diff < 0] = 0
      id <- ids[[idx]]
      x <- as.numeric(colnames(diff))
      y <- as.numeric(rownames(diff))

      p <- IOH_plot_ly_default('', 'Function Evaluations', 'f(x)')
      if (zero_transparant)
        diff[diff == 0] = NaN

      if (all(is.na(diff))) {
        p %<>% add_trace(
          z = 0,
          type = "contour",
          x = x,
          y = y,
          line = list(smoothing = 0),
          contours = list(
            start = ifelse(show_negatives, -1, 0),
            end = 1,
            coloring = 'fill',
            showlines = F
          ),
          colorscale = ifelse(show_negatives, 'BuRd_r' , 'Viridis'),
          reversescale = show_negatives,
          name = id
        )
      } else {
        p %<>% add_trace(
          z = diff,
          type = "contour",
          x = x,
          y = y,
          line = list(smoothing = 0),
          contours = list(
            start = ifelse(show_negatives, -1, 0),
            end = 1,
            coloring = 'fill',
            showlines = F
          ),
          colorscale = ifelse(show_negatives, 'BuRd_r' , 'Viridis'),
          reversescale = show_negatives,
          name = id
        )
      }
      p %<>% layout(yaxis = list(type = yscale, ticklen = 3))
      p %<>% layout(xaxis = list(type = xscale, ticklen = 3))
      if (show_colorbar) {
        show_colorbar <<- F
      } else {
        p %<>% hide_colorbar()
      }

      if (getOption("IOHanalyzer.annotation_x", 0.5) >= 0 &
          getOption("IOHanalyzer.annotation_y", 1) >= 0) {
        p %<>% layout(
          annotations = list(
            text = id,
            font = f2,
            showarrow = FALSE,
            xref = "paper",
            yref = "paper",
            x = getOption("IOHanalyzer.annotation_x", 0.5),
            y = getOption("IOHanalyzer.annotation_y", 1)
          )
        )
      }

      # p %<>% add_trace(x=x, y=fv_sum[ , .(max = max(max)), by = runtime]$max, color='white', type = "scatter", mode = "line", showlegend=F, alpha=0.4, name='max')
      # p %<>% add_trace(x=x, y=fv_sum[ , .(min = min(min)), by = runtime]$min, color='white', type = "scatter", mode = "line", showlegend=F, alpha=0.4, name='min')
      p
    })

    n_cols <- 1 + ceiling(length(matrices) / 10)
    n_rows <- ceiling(length(matrices) / n_cols)
    p <- subplot(
      ps,
      nrows = n_rows,
      titleX = T,
      titleY = T,
      margin = c(
        getOption("IOHanalyzer.margin_horizontal", 0.02),
        getOption("IOHanalyzer.margin_vertical", 0.02),
        getOption("IOHanalyzer.margin_horizontal", 0.02),
        getOption("IOHanalyzer.margin_vertical", 0.02)
      ),
      shareX = T,
      shareY = T
    )
    p

    return(p)
  }