In auto-optimization/iraceplot: Plots for Visualizing the Data Produced by the 'irace' Package

```{css, echo=FALSE} pre { white-space:pre !important; overflow-x: auto; max-width: 100%; } pre code { white-space:pre !important; overflow-x: auto; } .overflow { white-space:pre !important; overflow-x: auto; max-width: 100%; }

```r
knitr::opts_chunk$set(
  collapse = TRUE,
  comment = "#>"
)

library(iraceplot)
library(tidyr, quietly = TRUE)
library(dplyr, quietly = TRUE)
sections <- get0("sections", ifnotfound=list(experiments_matrix=TRUE,convergence=TRUE))
if (!exists("interactive_plots")) interactive_plots <- base::interactive()
# FIXME: Move show_table to the package so people can use it to create their
# own reports.
show_table <- function(x, row.names = FALSE, search = FALSE, lengthMenu = NULL,
                       searchHighlight = TRUE, dom='Blrtip', colorbar = NULL, style = list(), ...) {

  if (!interactive_plots) return(knitr::kable(x, row.names = row.names))

  if (search) dom <- sub("r", "fr", dom, fixed = TRUE)
  if (is.null(lengthMenu)) {
    pageLength <- NULL
  } else {
    pageLength <- lengthMenu[1]
    if (nrow(x) <= pageLength) dom <- gsub("[lip]", "", dom) # autoHideNavigation
  }
  columnDefs <- list(list(className = 'dt-right', targets = "_all"))
  tab <- DT::datatable(x, rownames = row.names,
                       class = 'compact row-border hover',
                       autoHideNavigation = TRUE,
                       extensions = c('Buttons', 'KeyTable'),
                       options = list(
                         keys = TRUE,
                         searchHighlight = search,
                         scrollX = TRUE,
                         pageLength=pageLength,
                         lengthMenu = lengthMenu,
                         dom = dom,
                         buttons = c('copy', 'csv'), #, 'colvis'),
                         columnDefs = columnDefs)) %>% DT::formatStyle(colnames(x), fontSize='85%')
  for (col in colorbar) {
    if(!(col %in% colnames(x)))
      stop("Column ", col, " not found in table: ", paste0(colnames(x), collapse=", "))
    if (all(is.na(x[, col]))) next # Don't add colorbar if all are NA.
    tab <- DT::formatStyle(tab, columns = col,
                           background = DT::styleColorBar(range(x[, col], na.rm=TRUE), 'lightblue'),
                           backgroundSize = '98% 88%',
                           backgroundRepeat = 'no-repeat',
                           backgroundPosition = 'center')
  }
  for (s in style) {
    tab <- do.call(DT::formatStyle, c(table=tab, s))
  }
  tab
}

best_elites <- as.character(irace_results$allElites[[length(irace_results$allElites)]])
irs <- irace_summarise(irace_results)
parameters <- irace_results$scenario$parameters

Scenario

Click to show

Parameters

knitr::kable(parameters_summarise(parameters))

Parameters tree

parameters_tree(parameters)

Parameters file

Click to show

irace::printParameters(parameters)

General information

• irace version: r {irs$version}
• Iterations: r {irs$n_iterations}
• Configurations: r {irs$n_configurations}
• Instances: r {irs$n_instances}
• Experiments: r {irs$n_experiments}
• Elite configurations: r {irs$n_elites}
• Soft restarts: r {irs$n_soft_restarts}
• Rejected configurations: r {irs$n_rejected}
• Running time (seconds): r {if (is.na(irs$time_wallclock)) "Unknown" else paste0(round(irs$time_cpu_user,0), " (CPU-user); ", round(irs$time_cpu_sys,0), " (CPU-sys); ", round(irs$time_wallclock,0), " (Wall-clock)")}
• Termination reason: r {irs$termination_reason}

By iteration

show_table(summarise_by_iteration(irace_results), lengthMenu = c(20, 50, 100),
           colorbar = c("configurations", "instances", "experiments", "elites"))

By instance

show_table(summarise_by_instance(irace_results), lengthMenu = c(20, 50, 100), search = TRUE,
           colorbar = c("experiments", "mean", "sd", "median", "min", "max"))
#columnDefs = list(list(targets = -1, searchable = FALSE))

Elite configurations

The final best configurations found by irace are:

show_table(irace_results$allConfigurations[best_elites, , drop=FALSE],
           lengthMenu = c(10,20),
           colorbar = names(which(parameters$types %in% c("i", "r") & !parameters$isFixed)))

Parallel coordinates visualization (only elites)

parallel_coord(irace_results)

Sampling model

The frequency of the parameter values sampled by irace:

sampling_frequency(irace_results)

Testing performance

• Number of elites tested: r {irace_results$scenario$testNbElites}
• Iteration elites tested: r {irace_results$scenario$testIterationElites}

Performance of the elite configurations on the test instances

if (has_testing_data(irace_results)) {
  best_results <- summarise_by_configuration(irace_results, instances = "test")
  show_table(best_results, lengthMenu = c(5, 10, 50),
             colorbar = c("mean", "sd", "median", "min", "max"))
} else {
  cat("No test instances given.\n")
}

Final elite configurations on the test instances

if (has_testing_data(irace_results)) {
  boxplot_test(irace_results, type = "best", interactive = interactive_plots)
} else {
  cat("No test instances given.\n")
}

if (has_testing_data(irace_results))
  boxplot_test(irace_results, type = "best", rpd = FALSE, interactive = interactive_plots)

Iteration elite configurations on the test instances

if (irace_results$scenario$testIterationElites && has_testing_data(irace_results)) {
   boxplot_test(irace_results, type = "all", show_points=FALSE, interactive = interactive_plots)
} else{
  cat("Iteration elites were not tested.\n")
}

Training performance

Performance of the final elite configurations on the training instances

best_results <- summarise_by_configuration(irace_results, elites_only = TRUE, instances = "train")
show_table(best_results, lengthMenu = c(10, 20, 50, 100),
           colorbar = c("mean", "sd", "median", "min", "max", "rank_mean", "rank_sd"))

Final elite configurations on the training instances

boxplot_training(irace_results, interactive = interactive_plots)

boxplot_training(irace_results, rpd = FALSE, interactive = interactive_plots)

```{asis, echo=irace_results$scenario$testIterationElites}

Iteration elite configurations on the training instances

```r
# FIXME: This should boxplot_training(irace_results, type="ibest")
boxplot_performance(experiments = irace_results$experiments,
                    allElites = lapply(irace_results$allElites, utils::head, irace_results$scenario$testNbElites),
                    type = "ibest", interactive = interactive_plots)

boxplot_performance(experiments = irace_results$experiments,
                    allElites = lapply(irace_results$allElites, utils::head, irace_results$scenario$testNbElites),
                    type = "ibest", rpd = FALSE, interactive = interactive_plots)

Races overview

plot_experiments_matrix(irace_results, interactive = interactive_plots)

``{asis, echo=!isTRUE(sections$experiments_matrix)} Disabled becausesections$experiments_matrix` is FALSE.

# Convergence plot

This is a simplified version of the visualization you can obtain with [acviz](https://github.com/souzamarcelo/acviz).


```r
configurations_display(irace_results, interactive = FALSE)

{asis, echo=!isTRUE(sections$convergence)} Disabled because `sections$convergence` is FALSE.

---

Report generated by iraceplot version r packageVersion("iraceplot").

auto-optimization/iraceplot documentation built on Nov. 29, 2024, 9:36 a.m.