R/dense_matrix_benchmark.R

In RHPCBenchmark: Benchmarks for High-Performance Computing Environments

################################################################################
# Copyright 2016 Indiana University
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################

#' Runs all of the dense matrix microbenchmarks
#'
#' \code{RunDenseMatrixBenchmark} runs all of the microbenchmarks for
#'   performance testing the dense matrix linear algebra kernels
#'
#' This function runs all of the dense matrix microbenchmarks defined in the
#' \code{microbenchmarks} input list for which the \code{active} field is 
#' set to TRUE.  For each microbenchmark, it attempts to create a
#' separate output file in CSV format containing the performance results for
#' each matrix tested by the microbenchmark.  The names of the output files
#' follow the format \code{benchmarkName}_\code{runIdentifier}.csv,
#' where \code{benchmarkName} is specified in the
#' \code{DenseMatrixMicrobenchmark} object of each microbenchmark, and
#' \code{runIdentifier} is an input parameter to this function.  If the file
#' already exists, the results will be appended to the existing file.  The
#' \code{microbenchmarks} input list
#' contains instances of the \code{DenseMatrixMicrobenchmark} class defining
#' each microbenchmark.  The default microbenchmarks are generated by the
#' function \code{\link{GetDenseMatrixDefaultMicrobenchmarks}}.  If the
#' linear algebra kernels are multithreaded, by linking to multithreaded
#' BLAS or LAPACK libraries for example, then the number of threads must
#' be retrievable from an environment variable which is set before execution of
#' the R programming environment.  The name of the environment variable
#' specifying the number of threads must be provided in the R HPC benchmark
#' environment variable R_BENCH_NUM_THREADS_VARIABLE.  This function will
#' retrieve the number of threads through R_BENCH_NUM_THREADS_VARIABLE so that
#' the number of threads can be printed to the results files and recorded in
#' data frames for reporting purposes.  This function utilizes the number of
#' threads only for reporting purposes and is not used by the benchmark to
#' effect the actual number of threads utilized by the kernels, as that is
#' assumed to be controlled by the numerical library.  An error exception will
#' be thrown if the environment variable R_BENCH_NUM_THREADS_VARIABLE and the
#' variable it is set to are not both set.
#'
#' @param runIdentifier a character string specifying the suffix to be
#'   appended to the base of the file name of the output CSV format files
#' @param resultsDirectory a character string specifying the directory
#'   where all of the CSV performance results files will be saved
#' @param microbenchmarks a list of \code{DenseMatrixMicrobenchmark} objects
#'   defining the microbenchmarks to execute as part of the dense matrix
#'   benchmark.  Default values are provided by the function
#'   \code{\link{GetDenseMatrixDefaultMicrobenchmarks}}.
#' @return a data frame containing the benchmark name, 
#'   user, system, and elapsed (wall clock) times of each performance trial
#'   for each microbenchmark
#'
#' @examples 
#' \dontrun{
#' # Set needed environment variables for multithreading.  Only single threading
#' # is used in the example.
#' #
#' # Note: The environment variables are usually set by the user before starting
#' #       the R programming environment; they are set here only to facilitate
#' #       a working example.  See the section on multithreading in the vignette
#' #       for further details.
#' Sys.setenv(R_BENCH_NUM_THREADS_VARIABLE="MKL_NUM_THREADS")
#' Sys.setenv(MKL_NUM_THREADS="1")
#' #
#' # Generate example microbechmarks that can be run in a few minutes; see
#' # the vignette for more involved examples.  The Cholesky factorization and
#' # matrix crossproduct microbenchmarks are performed in the example code
#' # below.
#' #
#' # Note: These microbenchmarks are different than the microbenchmarks
#' #       generated by \code{\link{GetDenseMatrixDefaultMicrobenchmarks}}.
#' #       They are chosen for their short run times and suitability for
#' #       example code. 
#' exampleMicrobenchmarks <- GetDenseMatrixExampleMicrobenchmarks()
#' # Set the output directory of the CSV summary results files
#' resultsDirectory <- "./DenseMatrixExampleOutput"
#' # Create the output directory
#' dir.create(resultsDirectory)
#' # Set an appropriate run identifier
#' runIdentifier <- "example"
#' resultsFrame <- RunDenseMatrixBenchmark(runIdentifier, resultsDirectory,
#'    microbenchmarks=exampleMicrobenchmarks)
#'
#' # This example runs all but the matrix transpose microbenchmarks.
#' exampleMicrobenchmarks[["transpose"]]$active <- FALSE
#' # Set an appropriate run identifier
#' runIdentifier <- "no_transpose"
#' exTransposeResultsFrame <- RunDenseMatrixBenchmark(runIdentifier,
#'   resultsDirectory, microbenchmarks=exampleMicrobenchmarks)
#'
#' # This example runs only the matrix-matrix multiplication microbenchmark,
#' # and it adds a larger matrix to test.
#' matMatMicrobenchmark <- list()
#' matMatMicrobenchmark[["matmat"]] <- GetDenseMatrixExampleMicrobenchmarks()[["matmat"]]
#' matMatMicrobenchmark[["matmat"]]$dimensionParameters <- as.integer(c(1000, 2000))
#' matMatMicrobenchmark[["matmat"]]$numberOfTrials <- as.integer(c(3, 3))
#' matMatMicrobenchmark[["matmat"]]$numberOfWarmupTrials <- as.integer(c(1, 1))
#' # Set an appropriate run identifier
#' runIdentifier <- "matmat"
#' matMatResults <- RunDenseMatrixBenchmark(runIdentifier, resultsDirectory,
#'    microbenchmarks=matMatMicrobenchmark)
#' }
#' 
#' @seealso \code{\link{GetDenseMatrixDefaultMicrobenchmarks}}
#'          \code{\link{GetDenseMatrixExampleMicrobenchmarks}}
#' @export
RunDenseMatrixBenchmark <- function(runIdentifier, resultsDirectory,
	microbenchmarks = GetDenseMatrixDefaultMicrobenchmarks()) {

   numberOfThreads <- GetNumberOfThreads()

   microbenchmarkResults <- NULL

   # Loop over all matrix kernel tests

   for (i in 1:length(microbenchmarks)) {
      if (microbenchmarks[[i]]$active) {
         benchmarkName <- microbenchmarks[[i]]$benchmarkName

         resultsFrame <- MicrobenchmarkDenseMatrixKernel(
            microbenchmarks[[i]], numberOfThreads, resultsDirectory,
            runIdentifier)
         microbenchmarkResults <- rbind(microbenchmarkResults, resultsFrame)
         invisible(gc())
      }
   }

   return (microbenchmarkResults)
}


#' Initializes the list of default dense matrix microbenchmarks
#'
#' \code{GetDenseMatrixDefaultMicrobenchmarks} defines the default dense
#' matrix microbenchmarks to be executed by the
#' \code{\link{RunDenseMatrixBenchmark}} function.  The current microbenchmarks
#' are Cholesky factorization, matrix cross product, matrix determinant,
#' eigendecomposition, linear solve with multiple right hand sides, least
#' squares fit, matrix deformation and transpose, matrix-matrix multiplication,
#' matrix-vector multiplication, QR decomposition, and singular value
#' decomposition.  See the documentation for the
#' \code{\link{DenseMatrixMicrobenchmark}} class for more details.
#'
#' @return a list of \code{DenseMatrixMicrobenchmark} objects defining the
#'   microbenchmarks to be executed.  The microbenchmarks appear in the order
#'   listed in the function description and are assigned the following names:
#'   cholesky, crossprod, determinant, eigen, solve, lsfit, deformtrans,
#'   transpose, matmat, matvec, qr, and svd. 
#' @seealso \code{\link{DenseMatrixMicrobenchmark}}
#' @export
GetDenseMatrixDefaultMicrobenchmarks <- function() {
   microbenchmarks <- list()

   # Define matrix kernel tests here

   # Cholesky factorization
   microbenchmarks[["cholesky"]] <- methods::new(
      "DenseMatrixMicrobenchmark",
      active = TRUE,
      benchmarkName = "cholesky",
      benchmarkDescription = "Dense matrix Cholesky factorization",
      dimensionParameters = as.integer(c(1000, 2000, 4000, 8000, 10000, 15000, 20000, 20000)),
      numberOfTrials = as.integer(c(20, 20, 10, 5, 5, 5, 5, 5)),
      numberOfWarmupTrials = as.integer(c(1, 1, 1, 1, 1, 1, 1, 1)),
      allocatorFunction = CholeskyAllocator,
      benchmarkFunction = CholeskyMicrobenchmark
   )

   # matrix cross product
   microbenchmarks[["crossprod"]] <- methods::new(
      "DenseMatrixMicrobenchmark",
      active = TRUE,
      benchmarkName = "crossprod",
      benchmarkDescription = "Dense matrix cross product",
      dimensionParameters = as.integer(c(1000, 2000, 4000, 8000, 10000, 15000, 20000, 20000)),
      numberOfTrials = as.integer(c(20, 20, 10, 5, 5, 5, 5, 5)),
      numberOfWarmupTrials = as.integer(c(1, 1, 1, 1, 1, 1, 1, 1)),
      allocatorFunction = CrossprodAllocator,
      benchmarkFunction = CrossprodMicrobenchmark
   )

   # Matrix deformation and transpose
   microbenchmarks[["deformtrans"]] <- methods::new(
      "DenseMatrixMicrobenchmark",
      active = TRUE,
      benchmarkName = "deformtrans",
      benchmarkDescription = "Dense matrix deformation and transpose",
      dimensionParameters = as.integer(c(1000, 2000, 4000, 8000, 10000, 15000, 20000, 20000)),
      numberOfTrials = as.integer(c(20, 20, 10, 5, 5, 5, 5, 5)),
      numberOfWarmupTrials = as.integer(c(1, 1, 1, 1, 1, 1, 1, 1)),
      allocatorFunction = DeformtransAllocator,
      benchmarkFunction = DeformtransMicrobenchmark
   )

   # matrix determinant
   microbenchmarks[["determinant"]] <- methods::new(
      "DenseMatrixMicrobenchmark",
      active = TRUE,
      benchmarkName = "determinant",
      benchmarkDescription = "Dense matrix determinant",
      dimensionParameters = as.integer(c(1000, 2000, 4000, 8000, 10000, 15000, 20000, 20000)),
      numberOfTrials = as.integer(c(20, 20, 10, 5, 5, 5, 5, 5)),
      numberOfWarmupTrials = as.integer(c(1, 1, 1, 1, 1, 1, 1, 1)),
      allocatorFunction = DeterminantAllocator,
      benchmarkFunction = DeterminantMicrobenchmark
   )

   # eigendecomposition
   microbenchmarks[["eigen"]] <- methods::new(
      "DenseMatrixMicrobenchmark",
      active = TRUE,
      benchmarkName = "eigendecomposition",
      benchmarkDescription = "Dense matrix eigendecomposition",
      dimensionParameters = as.integer(c(1000, 2000, 4000, 8000, 10000, 15000, 20000, 20000)),
      numberOfTrials = as.integer(c(10, 10, 5, 5, 5, 3, 3, 3)),
      numberOfWarmupTrials = as.integer(c(1, 1, 1, 1, 1, 1, 1, 1)),
      allocatorFunction = EigenAllocator,
      benchmarkFunction = EigenMicrobenchmark
   )

   # Least squares fit
   microbenchmarks[["lsfit"]] <- methods::new(
      "DenseMatrixMicrobenchmark",
      active = TRUE,
      benchmarkName = "lsfit",
      benchmarkDescription = "Dense least squares fit",
      dimensionParameters = as.integer(c(1000, 2000, 4000, 8000, 10000, 15000, 20000, 20000)),
      numberOfTrials = as.integer(c(20, 20, 10, 5, 5, 5, 5, 5)),
      numberOfWarmupTrials = as.integer(c(1, 1, 1, 1, 1, 1, 1, 1)),
      allocatorFunction = LsfitAllocator,
      benchmarkFunction = LsfitMicrobenchmark
   )

   # Matrix-matrix multiplication
   microbenchmarks[["matmat"]] <- methods::new(
      "DenseMatrixMicrobenchmark",
      active = TRUE,
      benchmarkName = "matmat",
      benchmarkDescription = "Dense matrix-matrix multiplication",
      dimensionParameters = as.integer(c(1000, 2000, 4000, 8000, 10000, 15000, 20000, 20000)),
      numberOfTrials = as.integer(c(20, 20, 10, 5, 5, 5, 5, 5)),
      numberOfWarmupTrials = as.integer(c(1, 1, 1, 1, 1, 1, 1, 1)),
      allocatorFunction = MatmatAllocator,
      benchmarkFunction = MatmatMicrobenchmark
   )

   # Matrix-vector multiplication
   microbenchmarks[["matvec"]] <- methods::new(
      "DenseMatrixMicrobenchmark",
      active = TRUE,
      benchmarkName = "matvec",
      benchmarkDescription = "Dense matrix-vector multiplication",
      dimensionParameters = as.integer(c(1000, 2000, 4000, 8000, 10000, 15000, 20000, 20000)),
      numberOfTrials = as.integer(c(20, 20, 10, 5, 5, 5, 5, 5)),
      numberOfWarmupTrials = as.integer(c(1, 1, 1, 1, 1, 1, 1, 1)),
      allocatorFunction = MatvecAllocator,
      benchmarkFunction = MatvecMicrobenchmark
   )

   # QR decomposition
   microbenchmarks[["qr"]] <- methods::new(
      "DenseMatrixMicrobenchmark",
      active = TRUE,
      benchmarkName = "qr",
      benchmarkDescription = "QR decomposition",
      dimensionParameters = as.integer(c(1000, 2000, 4000, 8000, 10000, 15000, 20000, 20000)),
      numberOfTrials = as.integer(c(20, 20, 10, 5, 5, 5, 5, 5)),
      numberOfWarmupTrials = as.integer(c(1, 1, 1, 1, 1, 1, 1, 1)),
      allocatorFunction = QrAllocator,
      benchmarkFunction = QrMicrobenchmark
   )

   # Linear solve with multiple right hand sides
   microbenchmarks[["solve"]] <- methods::new(
      "DenseMatrixMicrobenchmark",
      active = TRUE,
      benchmarkName = "solve",
      benchmarkDescription = "Dense linear solve with multiple r.h.s.",
      dimensionParameters = as.integer(c(1000, 2000, 4000, 8000, 10000, 15000, 20000, 20000)),
      numberOfTrials = as.integer(c(20, 20, 10, 5, 5, 5, 5, 5)),
      numberOfWarmupTrials = as.integer(c(1, 1, 1, 1, 1, 1, 1, 1)),
      allocatorFunction = SolveAllocator,
      benchmarkFunction = SolveMicrobenchmark
   )

   # Singular value decomposition
   microbenchmarks[["svd"]] <- methods::new(
      "DenseMatrixMicrobenchmark",
      active = TRUE,
      benchmarkName = "svd",
      benchmarkDescription = "Singular value decomposition",
      dimensionParameters = as.integer(c(1000, 2000, 4000, 8000, 10000, 15000, 20000, 20000)),
      numberOfTrials = as.integer(c(20, 20, 10, 5, 5, 5, 5, 5)),
      numberOfWarmupTrials = as.integer(c(1, 1, 1, 1, 1, 1, 1, 1)),
      allocatorFunction = SvdAllocator,
      benchmarkFunction = SvdMicrobenchmark
   )

   # Matrix transpose
   microbenchmarks[["transpose"]] <- methods::new(
      "DenseMatrixMicrobenchmark",
      active = TRUE,
      benchmarkName = "transpose",
      benchmarkDescription = "Dense matrix transpose",
      dimensionParameters = as.integer(c(1000, 2000, 4000, 8000, 10000, 15000, 20000, 20000)),
      numberOfTrials = as.integer(c(20, 20, 10, 5, 5, 5, 5, 5)),
      numberOfWarmupTrials = as.integer(c(1, 1, 1, 1, 1, 1, 1, 1)),
      allocatorFunction = TransposeAllocator,
      benchmarkFunction = TransposeMicrobenchmark
   )

   return (microbenchmarks)
}


#' Initializes the list of example dense matrix microbenchmarks
#'
#' \code{GetDenseMatrixExampleMicrobenchmarks} defines example dense
#' matrix microbenchmarks to be executed by the examples section
#' of the \code{\link{RunDenseMatrixBenchmark}} function.  The examples
#' are chosen so that they can run in a few minutes or less.
#'
#' @return a list of \code{DenseMatrixMicrobenchmark} objects defining the
#'   microbenchmarks to be executed.  Microbenchmarks for Cholesky factorization
#'   and matrix cross product are provided.
#'
#' @export
GetDenseMatrixExampleMicrobenchmarks <- function() {
   microbenchmarks <- list()

   # Define matrix kernel tests here

   # Cholesky factorization
   microbenchmarks[["cholesky"]] <- methods::new(
      "DenseMatrixMicrobenchmark",
      active = TRUE,
      benchmarkName = "cholesky",
      benchmarkDescription = "Dense matrix Cholesky factorization",
      dimensionParameters = as.integer(c(1000)),
      numberOfTrials = as.integer(c(3)),
      numberOfWarmupTrials = as.integer(c(1)),
      allocatorFunction = CholeskyAllocator,
      benchmarkFunction = CholeskyMicrobenchmark
   )

   # matrix cross product
   microbenchmarks[["crossprod"]] <- methods::new(
      "DenseMatrixMicrobenchmark",
      active = TRUE,
      benchmarkName = "crossprod",
      benchmarkDescription = "Dense matrix cross product",
      dimensionParameters = as.integer(c(1000)),
      numberOfTrials = as.integer(c(3)),
      numberOfWarmupTrials = as.integer(c(1)),
      allocatorFunction = CrossprodAllocator,
      benchmarkFunction = CrossprodMicrobenchmark
   )  

   # Matrix-matrix multiplication
   microbenchmarks[["matmat"]] <- methods::new(
      "DenseMatrixMicrobenchmark",
      active = TRUE,
      benchmarkName = "matmat",
      benchmarkDescription = "Dense matrix-matrix multiplication",
      dimensionParameters = as.integer(c(1000)),
      numberOfTrials = as.integer(c(3)),
      numberOfWarmupTrials = as.integer(c(1)),
      allocatorFunction = MatmatAllocator,
      benchmarkFunction = MatmatMicrobenchmark
   )

   # Matrix transpose
   microbenchmarks[["transpose"]] <- methods::new(
      "DenseMatrixMicrobenchmark",
      active = TRUE,
      benchmarkName = "transpose",
      benchmarkDescription = "Dense matrix transpose",
      dimensionParameters = as.integer(c(1000)),
      numberOfTrials = as.integer(c(3)),
      numberOfWarmupTrials = as.integer(c(1)),
      allocatorFunction = TransposeAllocator,
      benchmarkFunction = TransposeMicrobenchmark
   )
 
   return (microbenchmarks)
}