Nothing
SPOTMiscVignette
In SPOTMisc: Misc Extensions for the 'SPOT' Package
knitr::opts_chunk$set(
collapse = TRUE,
comment = "#>"
)
R Installation of Missing Packages
install.packages(c(
"benchmarkme", "callr", "emoa", "ggsci", "jsonlite", "keras", "magrittr", "mlr", "plotly", "RColorBrewer", "reticulate", "rpart.plot", "sensitivity", "smoof", "tensorflow", "tfdatasets"))
Python Installation
cd <project-dir>
Create a new virtual environment in a folder called python within the project directory using the following command:
virtualenv python
The virtualenv can be activated using the following command in a terminal:
source python/bin/activate
To verify that the correct version of python was activated the following command can be executed in a terminal:
which python
Install Python packages in the Environment
Python packages such as tensorflow-datasets, numpy, pandas, matplotlib, and other packages can be installed in the Python virtualenv by using
pip install:
pip install numpy pandas matplotlib tensorflow tensorflow-datasets
/Users/bartz/workspace/SPOT/python/bin/python -m pip install --upgrade pip
Install and Configure reticulate to use the Correct Python Version
Install the reticulate package using the following command in the R console (e.g., from within RStudio):
install.packages("reticulate")
To configure reticulate to point to the Python executable in the virtualenv python from above,
create a file in the project directory called .Rprofile with the following contents:
Sys.setenv(RETICULATE_PYTHON = "python/bin/python")
R (or RStudio) must be restarted for the setting to take effect.
To check that reticulate is configured for the correct version of Python the following command can be used in the R (or RStudio) console:
reticulate::py_config()
Installing Keras
To get started with Keras, the Keras R package, the core Keras library, and a backend tensor engine (such as TF) must be installed.
This can be done as follows from within R (or RStudio):
install.packages("tensorflow")
install.packages("keras")
library(keras)
install_keras()
This will provide you with a default CPU-based installation of Keras and TF.
To install a GPU-based version of the TF backend engine, the corresponding command reads as follows:
install_keras(tensorflow = "gpu")
## install.packages("devtools")
## devtools::install_github("r-lib/devtools")
url <- "http://owos.gm.fh-koeln.de:8055/bartz/spot.git"
devtools::install_git(url = url)
- Package version SPOT should be larger than
2.3.0.
library("SPOT")
packageVersion("SPOT")
With noise: (this takes some time)
# noisy objective
res1 <- spot(,function(x)funSphere(x)+rnorm(nrow(x)),c(-2,-3),c(1,2),
control=list(funEvals=40,noise=TRUE))
# noise with replicated evaluations
res2 <- spot(,function(x)funSphere(x)+rnorm(nrow(x)),c(-2,-3),c(1,2),
control=list(funEvals=40,noise=TRUE,replicates=2,
designControl=list(replicates=2)))
# and with OCBA
res3 <- spot(,function(x)funSphere(x)+rnorm(nrow(x)),c(-2,-3),c(1,2),
control=list(funEvals=40,noise=TRUE,replicates=2,OCBA=TRUE,OCBABudget=1,
designControl=list(replicates=2)))
# Check results with non-noisy function:
funSphere(res1$xbest)
funSphere(res2$xbest)
funSphere(res3$xbest)
Handling factor variables
Note: factors should be coded as integer values, i.e., 1,2,...,n
First, we create a test function with a factor variable:
braninFunctionFactor <- function (x) {
y <- (x[2] - 5.1/(4 * pi^2) * (x[1] ^2) + 5/pi * x[1] - 6)^2 +
10 * (1 - 1/(8 * pi)) * cos(x[1] ) + 10
if(x[3]==1)
y <- y +1
else if(x[3]==2)
y <- y -1
return(y)
}
Vectorize the test function.
objFun <- function(x){apply(x,1,braninFunctionFactor)}
Run spot.
set.seed(1)
res <- spot(fun=objFun,lower=c(-5,0,1),upper=c(10,15,3),
control=list(model=buildKriging,
types= c("numeric","numeric","factor"),
optimizer=optimLHD))
res$xbest
res$ybest
Run SPOT with logging
## run spot without log
res <- spot(fun = funSphere,
lower=c(0,0),
upper=c(100,100)
)
## run spot with log
funSphereLog <- function(x){
cbind(funSphere(x),x)
}
res2 <- spot(fun = funSphereLog,
lower=c(0,0),
upper=c(100,100)
)
res$logInfo
res2$logInfo
Keras/Tensorflow
First experiments (from the arxiv paper summer 2021)
exp01
library("SPOTMisc")
library("SPOT")
kerasConf <- getKerasConf()
lower <- c(1e-6, 1e-6, 16,0.6, 1e-9, 10, 6,0.4,0.99,1,1e-8)
upper <- c(0.5, 0.5, 512, 1.5, 1e-2, 50, 10,0.999,0.999,10,6e-8)
types <- c("numeric", "numeric", "integer", "numeric", "numeric",
"integer", "integer", "numeric", "numeric", "integer",
"numeric")
kerasConf$verbose <- 0
res <- spot(x = NULL,
fun = funKerasMnist,
lower = lower,
upper = upper,
control = list(funEvals=100,
noise = TRUE,
types = types,
plots = TRUE,
progress = TRUE,
seedFun = 1,
seedSPOT = 1),
kerasConf = kerasConf)
res01 <- res
# save(res01, file="exp01.RData")
exp02
library("SPOTMisc")
library("SPOT")
kerasConf <- getKerasConf()
lower <- c(1e-6, 1e-6, 16,0.6, 1e-9, 10, 6,0.4,0.99,1,1e-8)
upper <- c(0.5, 0.5, 512, 1.5, 1e-2, 50, 10,0.999,0.999,10,6e-8)
types <- c("numeric", "numeric", "integer", "numeric", "numeric",
"integer", "integer", "numeric", "numeric", "integer",
"numeric")
kerasConf$verbose <- 0
Ninit <- 3 * length(lower)
Rinit <- 3
replicates <- 3
res <- spot(x = NULL,
fun = funKerasMnist,
lower = lower,
upper = upper,
control = list(funEvals=20*length(lower),
noise = TRUE,
types = types,
replicates=replicates,
designControl = list(replicates = Rinit,
size = Ninit),
model = buildKriging,
optimizer = optimLBFGSB,
modelControl=list(target="ei"),
plots = TRUE,
progress = TRUE,
seedFun = 1,
seedSPOT = 1),
kerasConf = kerasConf)
res02 <- res
#save(res02, file="exp02.RData")
run002
#install.packages("~/workspace/SPOTMisc_1.2.26.tar.gz", repos=NULL, type="source")
#install.packages("~/workspace/SPOT_2.6.2.tar.gz", repos=NULL, type="source")
library("keras")
library("SPOT")
library("SPOTMisc")
packageVersion("SPOT")
packageVersion("SPOTMisc")
set.seed(123)
dat <- genCatsDogsData()
kerasConf <- getKerasConf()
kerasConf$trainData <- dat$trainData
kerasConf$validationData <- dat$validationData
kerasConf$testData <- dat$testData
lower <- c(1e-6, 1e-6, 16,0.6, 1e-9, 10, 6,0.4,0.99,1,1e-8)
upper <- c(0.5, 0.5, 512, 1.5, 1e-2, 50, 10,0.999,0.999,10,6e-8)
types <- c("numeric", "numeric", "integer", "numeric", "numeric",
"integer", "integer", "numeric", "numeric", "integer",
"numeric")
### First example: simple function call:
x <- matrix(lower, 1,)
funKerasTransferLearning(x, kerasConf = kerasConf)
kerasConf$verbose <- 1
res <- spot(x = NULL,
fun = funKerasTransferLearning,
lower = lower,
upper = upper,
control = list(funEvals=200,
noise = TRUE,
types = types,
plots = TRUE,
progress = TRUE,
seedFun = 1,
seedSPOT = 1),
kerasConf = kerasConf)
# save(res, file = "run002.RData")
keras/TF Experiments 2022
kerasMnist02
install.packages(c(
"benchmarkme", "callr", "emoa", "ggsci", "jsonlite", "keras", "magrittr", "mlr", "plotly", "RColorBrewer", "reticulate", "rpart.plot", "sensitivity", "smoof", "tensorflow", "tfdatasets"))
install.packages(
"~/workspace/spotmisc/packages/SPOT_2.7.0.tar.gz",
repos = NULL,
type = "source"
)
install.packages(
"~/workspace/spotmisc/packages/SPOTMisc_1.6.0.tar.gz",
repos = NULL,
type = "source"
)
library("SPOTMisc")
packageVersion("SPOTMisc")
library("SPOT")
packageVersion("SPOT")
mnist <- getMnistData()
# "dropout" = x[1]: 0.1--0.5
# "dropoutfact" = x[2]: 0.85--0.95
# "units" = x[3]: 7-9 (128--512)
# "unitsfact" = x[4]: 0.5--1.5
# "learning_rate" = x[5]: 1e-5--1e-2
# "epochs" = x[6]: 3-6 (10--50)
# "batchsize" = x[7]: 6-9 (64--512)
# "beta_1" = x[8]: 0.9--0.999
# "beta_2" = x[9]: 0.99--0.9999
# "layers" = x[10]: 1--5
# "epsilon" = x[11]: 1e-8--1e-8
lower <- c(0.1, 0.85, 7, 0.5, 1e-5, 3, 6, 0.9, 0.99, 1, 1e-8)
upper <- c(0.5, 0.95, 9, 1.5, 1e-2, 6, 9, 0.999, 0.9999, 5, 1e-6)
types <- rep("numeric", length(lower))
types[c(3, 6, 7)] <- "integer"
f2 <- function(x) {
2 ^ x
}
transformFun <- rep("identity", length(lower))
transformFun[c(3, 6, 7)] <- "f2"
kerasConf <- getKerasConf()
kerasConf$verbose <- 0
kerasConf$resDummy <- FALSE
Ninit <- 2 * length(lower)
Rinit <- 2
replicates <- 2
res <- spot(
x = NULL,
fun = funKerasMnist,
lower = lower,
upper = upper,
control = list(
funEvals = 20 * length(lower),
noise = TRUE,
types = types,
transformFun = transformFun,
replicates = replicates,
designControl = list(replicates = Rinit,
size = Ninit),
model = buildKriging,
optimizer = optimLBFGSB,
modelControl = list(target = "ei"),
plots = FALSE,
progress = TRUE,
seedFun = 1,
seedSPOT = 1
),
kerasConf = kerasConf,
data = mnist
)
save(res, file = paste0("resKerasMnist02", as.numeric(Sys.time()), ".RData"))
##
## Evaluation of the default hyperparameter setting (baseline)
library("SPOTMisc")
packageVersion("SPOTMisc")
##
source("~/workspace/spotmisc/R/funKerasMnist.R")
library("SPOT")
packageVersion("SPOT")
source("~/workspace/SPOT/R/spot.R")
source("~/workspace/SPOT/R/objectiveFunctionEvaluation.R")
mnist <- getMnistData()
# "dropout" = x[1],
# "dropoutfact" = x[2],
# "units" = x[3],
# "unitsfact" = x[4],
# "learning_rate" = x[5],
# "epochs" = x[6],
# "batchsize" = x[7],
# "beta_1" = x[8],
# "beta_2" = x[9],
# "layers" = x[10],
# "epsilon" = x[11]
lower <- c(0.4, 0.9, 8, 0.5, 0.001, 4, 3, 0.9, 0.999, 2, 1e-07)
upper <- lower
types <- rep("numeric", length(lower))
types[c(3, 6, 7)] <- "integer"
f2 <- function(x) {
2 ^ x
}
transformFun <- rep("identity", length(lower))
transformFun[c(3, 6, 7)] <- "f2"
#
kerasConf <- getKerasConf()
kerasConf$resDummy <- FALSE
kerasConf$verbose <- 0
kerasConf$returnValue <- "validationLoss"
kerasConf$naDummy <- 1234567890
kerasConf$clearSession <- TRUE
### Example: spot call to evaluate the default design
n <- 10
res <- spot(
x = NULL,
fun = funKerasMnist,
lower = lower,
upper = upper,
control = list(
funEvals = n,
transformFun = transformFun,
replicateResults = TRUE,
types = types,
seedFun = 1,
seedSPOT = 1,
verbosity = 0,
designControl=list(size=n)
),
kerasConf = kerasConf,
data = mnist
)
save(res, file = paste0("resKerasMnist02Default", as.numeric(Sys.time()), ".RData"))
KerasMnist03: same as 02, but training loss passed to optimizer
install.packages(c(
"benchmarkme", "callr", "emoa", "ggsci", "jsonlite", "keras", "magrittr", "mlr", "plotly", "RColorBrewer", "reticulate", "rpart.plot", "sensitivity", "smoof", "tensorflow", "tfdatasets"))
install.packages(
"~/workspace/spotmisc/packages/SPOT_2.7.0.tar.gz",
repos = NULL,
type = "source"
)
install.packages(
"~/workspace/spotmisc/packages/SPOTMisc_1.6.0.tar.gz",
repos = NULL,
type = "source"
)
library("SPOTMisc")
packageVersion("SPOTMisc")
library("SPOT")
packageVersion("SPOT")
mnist <- getMnistData()
# "dropout" = x[1]: 0.1--0.5
# "dropoutfact" = x[2]: 0.85--0.95
# "units" = x[3]: 7-9 (128--512)
# "unitsfact" = x[4]: 0.5--1.5
# "learning_rate" = x[5]: 1e-5--1e-2
# "epochs" = x[6]: 3-6 (10--50)
# "batchsize" = x[7]: 6-9 (64--512)
# "beta_1" = x[8]: 0.9--0.999
# "beta_2" = x[9]: 0.99--0.9999
# "layers" = x[10]: 1--5
# "epsilon" = x[11]: 1e-8--1e-8
lower <- c(0.1, 0.85, 7, 0.5, 1e-5, 3, 6, 0.9, 0.99, 1, 1e-8)
upper <- c(0.5, 0.95, 9, 1.5, 1e-2, 6, 9, 0.999, 0.9999, 5, 1e-6)
types <- rep("numeric", length(lower))
types[c(3, 6, 7)] <- "integer"
f2 <- function(x) {
2 ^ x
}
transformFun <- rep("identity", length(lower))
transformFun[c(3, 6, 7)] <- "f2"
kerasConf <- getKerasConf()
kerasConf$verbose <- 0
kerasConf$resDummy <- FALSE
kerasConf$returnValue <- "trainingLoss"
Ninit <- 2 * length(lower)
Rinit <- 2
replicates <- 2
res <- spot(
x = NULL,
fun = funKerasMnist,
lower = lower,
upper = upper,
control = list(
funEvals = 20 * length(lower),
noise = TRUE,
types = types,
transformFun = transformFun,
replicates = replicates,
designControl = list(replicates = Rinit,
size = Ninit),
model = buildKriging,
optimizer = optimLBFGSB,
modelControl = list(target = "ei"),
plots = FALSE,
progress = TRUE,
seedFun = 1,
seedSPOT = 1
),
kerasConf = kerasConf,
data = mnist
)
save(res, file = paste0("resKerasMnist03", as.numeric(Sys.time()),".RData"))
KerasMnist07: same as 02, but test loss passed to optimizer
install.packages(c(
"benchmarkme", "callr", "emoa", "ggsci", "jsonlite", "keras", "magrittr", "mlr", "plotly", "RColorBrewer", "reticulate", "rpart.plot", "sensitivity", "smoof", "tensorflow", "tfdatasets"))
install.packages(
"~/workspace/spotmisc/packages/SPOT_2.7.0.tar.gz",
repos = NULL,
type = "source"
)
install.packages(
"~/workspace/spotmisc/packages/SPOTMisc_1.6.0.tar.gz",
repos = NULL,
type = "source"
)
library("SPOTMisc")
packageVersion("SPOTMisc")
library("SPOT")
packageVersion("SPOT")
mnist <- getMnistData()
# "dropout" = x[1]: 0.1--0.5
# "dropoutfact" = x[2]: 0.85--0.95
# "units" = x[3]: 7-9 (128--512)
# "unitsfact" = x[4]: 0.5--1.5
# "learning_rate" = x[5]: 1e-5--1e-2
# "epochs" = x[6]: 3-6 (10--50)
# "batchsize" = x[7]: 6-9 (64--512)
# "beta_1" = x[8]: 0.9--0.999
# "beta_2" = x[9]: 0.99--0.9999
# "layers" = x[10]: 1--5
# "epsilon" = x[11]: 1e-8--1e-8
lower <- c(0.1, 0.85, 7, 0.5, 1e-5, 3, 6, 0.9, 0.99, 1, 1e-8)
upper <- c(0.5, 0.95, 9, 1.5, 1e-2, 6, 9, 0.999, 0.9999, 5, 1e-6)
types <- rep("numeric", length(lower))
types[c(3, 6, 7)] <- "integer"
f2 <- function(x) {
2 ^ x
}
transformFun <- rep("identity", length(lower))
transformFun[c(3, 6, 7)] <- "f2"
kerasConf <- getKerasConf()
kerasConf$verbose <- 0
kerasConf$resDummy <- FALSE
kerasConf$returnValue <- "testLoss"
Ninit <- 2 * length(lower)
Rinit <- 2
replicates <- 2
res <- spot(
x = NULL,
fun = funKerasMnist,
lower = lower,
upper = upper,
control = list(
funEvals = 20 * length(lower),
noise = TRUE,
types = types,
transformFun = transformFun,
replicates = replicates,
designControl = list(replicates = Rinit,
size = Ninit),
model = buildKriging,
optimizer = optimLBFGSB,
modelControl = list(target = "ei"),
plots = FALSE,
progress = TRUE,
seedFun = 1,
seedSPOT = 1
),
kerasConf = kerasConf,
data = mnist
)
save(res, file = paste0("resKerasMnist07", as.numeric(Sys.time()),".RData"))
KerasMnist08: same as 02, but negative validation accuracy passed to optimizer
install.packages(c(
"benchmarkme", "callr", "emoa", "ggsci", "jsonlite", "keras", "magrittr", "mlr", "plotly", "RColorBrewer", "reticulate", "rpart.plot", "sensitivity", "smoof", "tensorflow", "tfdatasets"))
install.packages(
"~/workspace/spotmisc/packages/SPOT_2.7.0.tar.gz",
repos = NULL,
type = "source"
)
install.packages(
"~/workspace/spotmisc/packages/SPOTMisc_1.6.0.tar.gz",
repos = NULL,
type = "source"
)
library("SPOTMisc")
packageVersion("SPOTMisc")
library("SPOT")
packageVersion("SPOT")
mnist <- getMnistData()
# "dropout" = x[1]: 0.1--0.5
# "dropoutfact" = x[2]: 0.85--0.95
# "units" = x[3]: 7-9 (128--512)
# "unitsfact" = x[4]: 0.5--1.5
# "learning_rate" = x[5]: 1e-5--1e-2
# "epochs" = x[6]: 3-6 (10--50)
# "batchsize" = x[7]: 6-9 (64--512)
# "beta_1" = x[8]: 0.9--0.999
# "beta_2" = x[9]: 0.99--0.9999
# "layers" = x[10]: 1--5
# "epsilon" = x[11]: 1e-8--1e-8
lower <- c(0.1, 0.85, 7, 0.5, 1e-5, 3, 6, 0.9, 0.99, 1, 1e-8)
upper <- c(0.5, 0.95, 9, 1.5, 1e-2, 6, 9, 0.999, 0.9999, 5, 1e-6)
types <- rep("numeric", length(lower))
types[c(3, 6, 7)] <- "integer"
f2 <- function(x) {
2 ^ x
}
transformFun <- rep("identity", length(lower))
transformFun[c(3, 6, 7)] <- "f2"
kerasConf <- getKerasConf()
kerasConf$verbose <- 0
kerasConf$resDummy <- FALSE
kerasConf$returnValue <- "negValidationAccuracy"
Ninit <- 2 * length(lower)
Rinit <- 2
replicates <- 2
res <- spot(
x = NULL,
fun = funKerasMnist,
lower = lower,
upper = upper,
control = list(
funEvals = 20 * length(lower),
noise = TRUE,
types = types,
transformFun = transformFun,
replicates = replicates,
designControl = list(replicates = Rinit,
size = Ninit),
model = buildKriging,
optimizer = optimLBFGSB,
modelControl = list(target = "ei"),
plots = FALSE,
progress = TRUE,
seedFun = 1,
seedSPOT = 1
),
kerasConf = kerasConf,
data = mnist
)
save(res, file = paste0("resKerasMnist08", as.numeric(Sys.time()),".RData"))
KerasMnist09: same as 02, but negative training accuracy passed to optimizer
install.packages(c(
"benchmarkme", "callr", "emoa", "ggsci", "jsonlite", "keras", "magrittr", "mlr", "plotly", "RColorBrewer", "reticulate", "rpart.plot", "sensitivity", "smoof", "tensorflow", "tfdatasets"))
install.packages(
"~/workspace/spotmisc/packages/SPOT_2.7.0.tar.gz",
repos = NULL,
type = "source"
)
install.packages(
"~/workspace/spotmisc/packages/SPOTMisc_1.6.0.tar.gz",
repos = NULL,
type = "source"
)
library("SPOTMisc")
packageVersion("SPOTMisc")
library("SPOT")
packageVersion("SPOT")
mnist <- getMnistData()
# "dropout" = x[1]: 0.1--0.5
# "dropoutfact" = x[2]: 0.85--0.95
# "units" = x[3]: 7-9 (128--512)
# "unitsfact" = x[4]: 0.5--1.5
# "learning_rate" = x[5]: 1e-5--1e-2
# "epochs" = x[6]: 3-6 (10--50)
# "batchsize" = x[7]: 6-9 (64--512)
# "beta_1" = x[8]: 0.9--0.999
# "beta_2" = x[9]: 0.99--0.9999
# "layers" = x[10]: 1--5
# "epsilon" = x[11]: 1e-8--1e-8
lower <- c(0.1, 0.85, 7, 0.5, 1e-5, 3, 6, 0.9, 0.99, 1, 1e-8)
upper <- c(0.5, 0.95, 9, 1.5, 1e-2, 6, 9, 0.999, 0.9999, 5, 1e-6)
types <- rep("numeric", length(lower))
types[c(3, 6, 7)] <- "integer"
f2 <- function(x) {
2 ^ x
}
transformFun <- rep("identity", length(lower))
transformFun[c(3, 6, 7)] <- "f2"
kerasConf <- getKerasConf()
kerasConf$verbose <- 0
kerasConf$resDummy <- FALSE
kerasConf$returnValue <- "negTrainingAccuracy"
Ninit <- 2 * length(lower)
Rinit <- 2
replicates <- 2
res <- spot(
x = NULL,
fun = funKerasMnist,
lower = lower,
upper = upper,
control = list(
funEvals = 20 * length(lower),
noise = TRUE,
types = types,
transformFun = transformFun,
replicates = replicates,
designControl = list(replicates = Rinit,
size = Ninit),
model = buildKriging,
optimizer = optimLBFGSB,
modelControl = list(target = "ei"),
plots = FALSE,
progress = TRUE,
seedFun = 1,
seedSPOT = 1
),
kerasConf = kerasConf,
data = mnist
)
save(res, file = paste0("resKerasMnist09", as.numeric(Sys.time()),".RData"))
KerasMnist10: same as 02, but negative test accuracy passed to optimizer
install.packages(c(
"benchmarkme", "callr", "emoa", "ggsci", "jsonlite", "keras", "magrittr", "mlr", "plotly", "RColorBrewer", "reticulate", "rpart.plot", "sensitivity", "smoof", "tensorflow", "tfdatasets"))
install.packages(
"~/workspace/spotmisc/packages/SPOT_2.7.0.tar.gz",
repos = NULL,
type = "source"
)
install.packages(
"~/workspace/spotmisc/packages/SPOTMisc_1.6.0.tar.gz",
repos = NULL,
type = "source"
)
library("SPOTMisc")
packageVersion("SPOTMisc")
library("SPOT")
packageVersion("SPOT")
mnist <- getMnistData()
# "dropout" = x[1]: 0.1--0.5
# "dropoutfact" = x[2]: 0.85--0.95
# "units" = x[3]: 7-9 (128--512)
# "unitsfact" = x[4]: 0.5--1.5
# "learning_rate" = x[5]: 1e-5--1e-2
# "epochs" = x[6]: 3-6 (10--50)
# "batchsize" = x[7]: 6-9 (64--512)
# "beta_1" = x[8]: 0.9--0.999
# "beta_2" = x[9]: 0.99--0.9999
# "layers" = x[10]: 1--5
# "epsilon" = x[11]: 1e-8--1e-8
lower <- c(0.1, 0.85, 7, 0.5, 1e-5, 3, 6, 0.9, 0.99, 1, 1e-8)
upper <- c(0.5, 0.95, 9, 1.5, 1e-2, 6, 9, 0.999, 0.9999, 5, 1e-6)
types <- rep("numeric", length(lower))
types[c(3, 6, 7)] <- "integer"
f2 <- function(x) {
2 ^ x
}
transformFun <- rep("identity", length(lower))
transformFun[c(3, 6, 7)] <- "f2"
kerasConf <- getKerasConf()
kerasConf$verbose <- 0
kerasConf$resDummy <- FALSE
kerasConf$returnValue <- "negTestAccuracy"
Ninit <- 2 * length(lower)
Rinit <- 2
replicates <- 2
res <- spot(
x = NULL,
fun = funKerasMnist,
lower = lower,
upper = upper,
control = list(
funEvals = 20 * length(lower),
noise = TRUE,
types = types,
transformFun = transformFun,
replicates = replicates,
designControl = list(replicates = Rinit,
size = Ninit),
model = buildKriging,
optimizer = optimLBFGSB,
modelControl = list(target = "ei"),
plots = FALSE,
progress = TRUE,
seedFun = 1,
seedSPOT = 1
),
kerasConf = kerasConf,
data = mnist
)
save(res, file = paste0("resKerasMnist10", as.numeric(Sys.time()),".RData"))
Transfer Learning
resKerasTransferLearning04
library("SPOTMisc")
library("SPOT")
# Hyperparameters:
# "dropout" = x[1],
# "learning_rate" = x[2],
# "epochs" = x[3],
# "beta_1" = x[4],
# "beta_2" = x[5],
# "epsilon" = x[6],
# "optimizer" = x[7]
lower <- c(1e-6, 1e-6, 2, 0.8, 0.8, 1e-9, 1)
upper <- c(0.2, 1e-2, 5, 0.99, 0.9999, 1e-3, 2)
types <- c("numeric", "numeric", "integer", "numeric", "numeric",
"numeric", "factor")
kerasConf <- getKerasConf()
kerasConf$verbose <- 1
kerasConf$resDummy <- FALSE
kerasConf$naDummy <- 1234567890
res <- spot(x = NULL,
fun = funKerasTransferLearning,
lower = lower,
upper = upper,
control = list(funEvals=50,
model=buildKriging,
noise = TRUE,
types = types,
optimizer=optimDE,
plots = FALSE,
progress = TRUE,
seedFun = 1,
seedSPOT = 1,
verbosity = 1),
kerasConf = kerasConf
)
save(res, file = paste0("resKerasTransferLearning04", as.numeric(Sys.time()),".RData"))
resKerasTransferLearning05
library("SPOTMisc")
library("SPOT")
kerasConf <- getKerasConf()
# Hyperparameters:
# "dropout" = x[1],
# "learning_rate" = x[2],
# "epochs" = x[3],
# "beta_1" = x[4],
# "beta_2" = x[5],
# "epsilon" = x[6],
# "optimizer" = x[7]
lower <- c(1e-6, 1e-6, 2, 0.8, 0.8, 1e-9, 1)
upper <- c(0.2, 1e-2, 5, 0.99, 0.9999, 1e-3, 2)
types <- c("numeric", "numeric", "integer", "numeric", "numeric",
"numeric", "factor")
### Example: spot call with extended verbosity and negative val accuracy:
kerasConf$verbose <- 0
kerasConf$returnValue <- "negValidationAccuracy"
res <- spot(x = NULL,
fun = funKerasTransferLearning,
lower = lower,
upper = upper,
control = list(funEvals=50,
model=buildKriging,
noise = TRUE,
types = types,
optimizer=optimDE,
plots = FALSE,
progress = TRUE,
seedFun = 1,
seedSPOT = 1,
verbosity = 1),
kerasConf = kerasConf
)
save(res, file = paste0("resKerasTransferLearning05", as.numeric(Sys.time()),".RData"))
resKerasTransferLearning06
- Evaluation of the default hyperparameter setting (baseline)
library("SPOTMisc")
library("SPOT")
kerasConf <- getKerasConf()
# Hyperparameters:
# "dropout" = x[1],
# "learning_rate" = x[2],
# "epochs" = x[3],
# "beta_1" = x[4],
# "beta_2" = x[5],
# "epsilon" = x[6],
# "optimizer" = x[7]
replicates <- 1
lower <- c(0.2, 1e-5, 5, 0.9, 0.999, 1e-7, 1)
upper <- c(0.2, 1e-5, 5, 0.9, 0.999, 1e-7, 1)
types <- c("numeric", "numeric", "integer", "numeric", "numeric",
"numeric", "factor")
### Example: spot call to evaluate the default design
kerasConf$verbose <- 0
kerasConf$returnValue <- "validationLoss"
kerasConf$naDummy <- 1234567890
res <- spot(x = NULL,
fun = funKerasTransferLearning,
lower = lower,
upper = upper,
control = list(funEvals=10,
model=buildKriging,
noise = TRUE,
types = types,
optimizer = optimDE,
plots = FALSE,
progress = TRUE,
designControl=list(replicates=replicates),
seedFun = 1,
seedSPOT = 1,
verbosity = 1),
kerasConf = kerasConf
)
save(res, file = paste0("resKerasTransferLearning06", as.numeric(Sys.time()),".RData"))
kerasMnist11: 02 with Multi Start
install.packages(c(
"benchmarkme", "callr", "emoa", "ggsci", "jsonlite", "keras", "magrittr", "mlr", "plotly", "RColorBrewer", "reticulate", "rpart.plot", "sensitivity", "smoof", "tensorflow", "tfdatasets"))
install.packages(
"~/workspace/spotmisc/packages/SPOT_2.8.4.tar.gz",
repos = NULL,
type = "source"
)
install.packages(
"~/workspace/spotmisc/packages/SPOTMisc_1.8.0.tar.gz",
repos = NULL,
type = "source"
)
library("SPOTMisc")
packageVersion("SPOTMisc")
library("SPOT")
packageVersion("SPOT")
mnist <- getMnistData()
# "dropout" = x[1]: 0.1--0.5
# "dropoutfact" = x[2]: 0.85--0.95
# "units" = x[3]: 7-9 (128--512)
# "unitsfact" = x[4]: 0.5--1.5
# "learning_rate" = x[5]: 1e-5--1e-2
# "epochs" = x[6]: 3-6 (10--50)
# "batchsize" = x[7]: 6-9 (64--512)
# "beta_1" = x[8]: 0.9--0.999
# "beta_2" = x[9]: 0.99--0.9999
# "layers" = x[10]: 1--5
# "epsilon" = x[11]: 1e-8--1e-8
lower <- c(0.1, 0.85, 7, 0.5, 1e-5, 3, 6, 0.9, 0.99, 1, 1e-8)
upper <- c(0.5, 0.95, 9, 1.5, 1e-2, 6, 9, 0.999, 0.9999, 5, 1e-6)
types <- rep("numeric", length(lower))
types[c(3, 6, 7)] <- "integer"
f2 <- function(x) {
2 ^ x
}
transformFun <- rep("identity", length(lower))
transformFun[c(3, 6, 7)] <- "f2"
kerasConf <- getKerasConf()
kerasConf$verbose <- 0
kerasConf$resDummy <- TRUE
Ninit <- 2 * length(lower)
Rinit <- 2
replicates <- 2
res <- spot(
x = NULL,
fun = funKerasMnist,
lower = lower,
upper = upper,
control = list(
funEvals = 20 * length(lower),
multiStart = 2,
noise = TRUE,
types = types,
transformFun = transformFun,
replicates = replicates,
designControl = list(replicates = Rinit,
size = Ninit),
model = buildKriging,
optimizer = optimDE,
modelControl = list(target = "ei"),
plots = FALSE,
progress = TRUE,
seedFun = 1,
seedSPOT = 1,
verbosity=0
),
kerasConf = kerasConf,
data = mnist
)
save(res, file = paste0("resKerasMnist11", as.numeric(Sys.time()), ".RData"))
Census Data
# install.packages("~/workspace/book/packages/SPOTMisc_1.8.2.tar.gz", repos = NULL, type = "source")
# install.packages("~/workspace/book/packages/SPOT_2.8.4.tar.gz", repos = NULL, type = "source")
library("SPOTMisc")
if (packageVersion("SPOTMisc") < "1.8.0") stop("Please update 'SPOT'")
library("SPOT")
if (packageVersion("SPOT") < "2.8.4") stop("Please update 'SPOT'")
# "dropout" = x[1]: 0.1--0.5
# "dropoutfact" = x[2]: 1-1
# "units" = x[3]: 1-1
# "unitsfact" = x[4]: 1-1
# "learning_rate" = x[5]: 1e-5--1e-2
# "epochs" = x[6]: 3-6
# "beta_1" = x[7]: 0.9--0.999
# "beta_2" = x[8]: 0.99--0.9999
# "layers" = x[9]: 1-1
# "epsilon" = x[10]: 1e-8--1e-6
lower <- c(0.1, 1, 1, 1, 1e-5, 3, 0.9, 0.99, 1, 1e-8)
upper <- c(0.5, 1, 1, 1, 1e-2, 6, 0.999, 0.9999, 1, 1e-6)
types <- rep("numeric", length(lower))
types[c(3, 4, 6, 9)] <- "integer"
f2 <- function(x) {
2 ^ x
}
transformFun <- rep("identity", length(lower))
transformFun[c(3, 4, 6, 9)] <- "f2"
kerasConf <- getKerasConf()
kerasConf$verbose <- 0
kerasConf$resDummy <- FALSE
Ninit <- 2 * length(lower)
Rinit <- 2
replicates <- 2
task.type <- "classif"
nobs <- 1e4 # max: 229285
nfactors = "high"
nnumericals <- "high"
cardinality = "high"
data.seed <- 1
tuner.seed <- 1
timebudget <- 3600
timeout <- timebudget/20
data <- getCensusTrainValTestData(task.type=task.type,
nobs=nobs,
nfactors = nfactors,
nnumericals = nnumericals,
cardinality = cardinality,
data.seed =1)
specCensusPrep <- censusDataPrep(data=data, batch_size = 32)
res <- spot(
x = NULL,
fun = funKerasCensus,
lower = lower,
upper = upper,
control = list(
funEvals = 20 * length(lower),
multiStart = 2,
noise = TRUE,
types = types,
transformFun = transformFun,
replicates = replicates,
designControl = list(replicates = Rinit,
size = Ninit),
model = buildKriging,
optimizer = optimDE,
modelControl = list(target = "y"),
plots = FALSE,
progress = TRUE,
seedFun = 1,
seedSPOT = 1,
verbosity=0
),
kerasConf = kerasConf,
specCensusPrep =specCensusPrep
)
save(res, file = paste0("rescs3Census01", as.numeric(Sys.time()), ".RData"))
Try the SPOTMisc package in your browser
Any scripts or data that you put into this service are public.
SPOTMisc documentation built on Sept. 5, 2022, 5:06 p.m.
R Package Documentation
Browse R Packages
We want your feedback!
Note that we can't provide technical support on individual packages. You should contact the package authors for that.
knitr::opts_chunk$set( collapse = TRUE, comment = "#>" )
R Installation of Missing Packages
install.packages(c( "benchmarkme", "callr", "emoa", "ggsci", "jsonlite", "keras", "magrittr", "mlr", "plotly", "RColorBrewer", "reticulate", "rpart.plot", "sensitivity", "smoof", "tensorflow", "tfdatasets"))
Python Installation
cd <project-dir>
Create a new virtual environment in a folder called python within the project directory using the following command:
virtualenv python
The virtualenv can be activated using the following command in a terminal:
source python/bin/activate
To verify that the correct version of python was activated the following command can be executed in a terminal:
which python
Install Python packages in the Environment
Python packages such as tensorflow-datasets, numpy, pandas, matplotlib, and other packages can be installed in the Python virtualenv by using pip install:
pip install numpy pandas matplotlib tensorflow tensorflow-datasets
/Users/bartz/workspace/SPOT/python/bin/python -m pip install --upgrade pip
Install and Configure reticulate to use the Correct Python Version
Install the reticulate package using the following command in the R console (e.g., from within RStudio):
install.packages("reticulate")
To configure reticulate to point to the Python executable in the virtualenv python from above, create a file in the project directory called .Rprofile with the following contents:
Sys.setenv(RETICULATE_PYTHON = "python/bin/python")
R (or RStudio) must be restarted for the setting to take effect. To check that reticulate is configured for the correct version of Python the following command can be used in the R (or RStudio) console:
reticulate::py_config()
Installing Keras
To get started with Keras, the Keras R package, the core Keras library, and a backend tensor engine (such as TF) must be installed. This can be done as follows from within R (or RStudio):
install.packages("tensorflow") install.packages("keras") library(keras) install_keras()
This will provide you with a default CPU-based installation of Keras and TF. To install a GPU-based version of the TF backend engine, the corresponding command reads as follows:
install_keras(tensorflow = "gpu")
## install.packages("devtools") ## devtools::install_github("r-lib/devtools") url <- "http://owos.gm.fh-koeln.de:8055/bartz/spot.git" devtools::install_git(url = url)
- Package version SPOT should be larger than
2.3.0.
library("SPOT") packageVersion("SPOT")
With noise: (this takes some time)
# noisy objective res1 <- spot(,function(x)funSphere(x)+rnorm(nrow(x)),c(-2,-3),c(1,2), control=list(funEvals=40,noise=TRUE)) # noise with replicated evaluations res2 <- spot(,function(x)funSphere(x)+rnorm(nrow(x)),c(-2,-3),c(1,2), control=list(funEvals=40,noise=TRUE,replicates=2, designControl=list(replicates=2))) # and with OCBA res3 <- spot(,function(x)funSphere(x)+rnorm(nrow(x)),c(-2,-3),c(1,2), control=list(funEvals=40,noise=TRUE,replicates=2,OCBA=TRUE,OCBABudget=1, designControl=list(replicates=2))) # Check results with non-noisy function: funSphere(res1$xbest) funSphere(res2$xbest) funSphere(res3$xbest)
Handling factor variables
Note: factors should be coded as integer values, i.e., 1,2,...,n First, we create a test function with a factor variable:
braninFunctionFactor <- function (x) { y <- (x[2] - 5.1/(4 * pi^2) * (x[1] ^2) + 5/pi * x[1] - 6)^2 + 10 * (1 - 1/(8 * pi)) * cos(x[1] ) + 10 if(x[3]==1) y <- y +1 else if(x[3]==2) y <- y -1 return(y) }
Vectorize the test function.
objFun <- function(x){apply(x,1,braninFunctionFactor)}
Run spot.
set.seed(1) res <- spot(fun=objFun,lower=c(-5,0,1),upper=c(10,15,3), control=list(model=buildKriging, types= c("numeric","numeric","factor"), optimizer=optimLHD)) res$xbest res$ybest
Run SPOT with logging
## run spot without log res <- spot(fun = funSphere, lower=c(0,0), upper=c(100,100) ) ## run spot with log funSphereLog <- function(x){ cbind(funSphere(x),x) } res2 <- spot(fun = funSphereLog, lower=c(0,0), upper=c(100,100) ) res$logInfo res2$logInfo
Keras/Tensorflow
First experiments (from the arxiv paper summer 2021)
exp01
library("SPOTMisc") library("SPOT") kerasConf <- getKerasConf() lower <- c(1e-6, 1e-6, 16,0.6, 1e-9, 10, 6,0.4,0.99,1,1e-8) upper <- c(0.5, 0.5, 512, 1.5, 1e-2, 50, 10,0.999,0.999,10,6e-8) types <- c("numeric", "numeric", "integer", "numeric", "numeric", "integer", "integer", "numeric", "numeric", "integer", "numeric") kerasConf$verbose <- 0 res <- spot(x = NULL, fun = funKerasMnist, lower = lower, upper = upper, control = list(funEvals=100, noise = TRUE, types = types, plots = TRUE, progress = TRUE, seedFun = 1, seedSPOT = 1), kerasConf = kerasConf) res01 <- res # save(res01, file="exp01.RData")
exp02
library("SPOTMisc") library("SPOT") kerasConf <- getKerasConf() lower <- c(1e-6, 1e-6, 16,0.6, 1e-9, 10, 6,0.4,0.99,1,1e-8) upper <- c(0.5, 0.5, 512, 1.5, 1e-2, 50, 10,0.999,0.999,10,6e-8) types <- c("numeric", "numeric", "integer", "numeric", "numeric", "integer", "integer", "numeric", "numeric", "integer", "numeric") kerasConf$verbose <- 0 Ninit <- 3 * length(lower) Rinit <- 3 replicates <- 3 res <- spot(x = NULL, fun = funKerasMnist, lower = lower, upper = upper, control = list(funEvals=20*length(lower), noise = TRUE, types = types, replicates=replicates, designControl = list(replicates = Rinit, size = Ninit), model = buildKriging, optimizer = optimLBFGSB, modelControl=list(target="ei"), plots = TRUE, progress = TRUE, seedFun = 1, seedSPOT = 1), kerasConf = kerasConf) res02 <- res #save(res02, file="exp02.RData")
run002
#install.packages("~/workspace/SPOTMisc_1.2.26.tar.gz", repos=NULL, type="source") #install.packages("~/workspace/SPOT_2.6.2.tar.gz", repos=NULL, type="source") library("keras") library("SPOT") library("SPOTMisc") packageVersion("SPOT") packageVersion("SPOTMisc") set.seed(123) dat <- genCatsDogsData() kerasConf <- getKerasConf() kerasConf$trainData <- dat$trainData kerasConf$validationData <- dat$validationData kerasConf$testData <- dat$testData lower <- c(1e-6, 1e-6, 16,0.6, 1e-9, 10, 6,0.4,0.99,1,1e-8) upper <- c(0.5, 0.5, 512, 1.5, 1e-2, 50, 10,0.999,0.999,10,6e-8) types <- c("numeric", "numeric", "integer", "numeric", "numeric", "integer", "integer", "numeric", "numeric", "integer", "numeric") ### First example: simple function call: x <- matrix(lower, 1,) funKerasTransferLearning(x, kerasConf = kerasConf) kerasConf$verbose <- 1 res <- spot(x = NULL, fun = funKerasTransferLearning, lower = lower, upper = upper, control = list(funEvals=200, noise = TRUE, types = types, plots = TRUE, progress = TRUE, seedFun = 1, seedSPOT = 1), kerasConf = kerasConf) # save(res, file = "run002.RData")
keras/TF Experiments 2022
kerasMnist02
install.packages(c( "benchmarkme", "callr", "emoa", "ggsci", "jsonlite", "keras", "magrittr", "mlr", "plotly", "RColorBrewer", "reticulate", "rpart.plot", "sensitivity", "smoof", "tensorflow", "tfdatasets")) install.packages( "~/workspace/spotmisc/packages/SPOT_2.7.0.tar.gz", repos = NULL, type = "source" ) install.packages( "~/workspace/spotmisc/packages/SPOTMisc_1.6.0.tar.gz", repos = NULL, type = "source" ) library("SPOTMisc") packageVersion("SPOTMisc") library("SPOT") packageVersion("SPOT") mnist <- getMnistData() # "dropout" = x[1]: 0.1--0.5 # "dropoutfact" = x[2]: 0.85--0.95 # "units" = x[3]: 7-9 (128--512) # "unitsfact" = x[4]: 0.5--1.5 # "learning_rate" = x[5]: 1e-5--1e-2 # "epochs" = x[6]: 3-6 (10--50) # "batchsize" = x[7]: 6-9 (64--512) # "beta_1" = x[8]: 0.9--0.999 # "beta_2" = x[9]: 0.99--0.9999 # "layers" = x[10]: 1--5 # "epsilon" = x[11]: 1e-8--1e-8 lower <- c(0.1, 0.85, 7, 0.5, 1e-5, 3, 6, 0.9, 0.99, 1, 1e-8) upper <- c(0.5, 0.95, 9, 1.5, 1e-2, 6, 9, 0.999, 0.9999, 5, 1e-6) types <- rep("numeric", length(lower)) types[c(3, 6, 7)] <- "integer" f2 <- function(x) { 2 ^ x } transformFun <- rep("identity", length(lower)) transformFun[c(3, 6, 7)] <- "f2" kerasConf <- getKerasConf() kerasConf$verbose <- 0 kerasConf$resDummy <- FALSE Ninit <- 2 * length(lower) Rinit <- 2 replicates <- 2 res <- spot( x = NULL, fun = funKerasMnist, lower = lower, upper = upper, control = list( funEvals = 20 * length(lower), noise = TRUE, types = types, transformFun = transformFun, replicates = replicates, designControl = list(replicates = Rinit, size = Ninit), model = buildKriging, optimizer = optimLBFGSB, modelControl = list(target = "ei"), plots = FALSE, progress = TRUE, seedFun = 1, seedSPOT = 1 ), kerasConf = kerasConf, data = mnist ) save(res, file = paste0("resKerasMnist02", as.numeric(Sys.time()), ".RData")) ## ## Evaluation of the default hyperparameter setting (baseline) library("SPOTMisc") packageVersion("SPOTMisc") ## source("~/workspace/spotmisc/R/funKerasMnist.R") library("SPOT") packageVersion("SPOT") source("~/workspace/SPOT/R/spot.R") source("~/workspace/SPOT/R/objectiveFunctionEvaluation.R") mnist <- getMnistData() # "dropout" = x[1], # "dropoutfact" = x[2], # "units" = x[3], # "unitsfact" = x[4], # "learning_rate" = x[5], # "epochs" = x[6], # "batchsize" = x[7], # "beta_1" = x[8], # "beta_2" = x[9], # "layers" = x[10], # "epsilon" = x[11] lower <- c(0.4, 0.9, 8, 0.5, 0.001, 4, 3, 0.9, 0.999, 2, 1e-07) upper <- lower types <- rep("numeric", length(lower)) types[c(3, 6, 7)] <- "integer" f2 <- function(x) { 2 ^ x } transformFun <- rep("identity", length(lower)) transformFun[c(3, 6, 7)] <- "f2" # kerasConf <- getKerasConf() kerasConf$resDummy <- FALSE kerasConf$verbose <- 0 kerasConf$returnValue <- "validationLoss" kerasConf$naDummy <- 1234567890 kerasConf$clearSession <- TRUE ### Example: spot call to evaluate the default design n <- 10 res <- spot( x = NULL, fun = funKerasMnist, lower = lower, upper = upper, control = list( funEvals = n, transformFun = transformFun, replicateResults = TRUE, types = types, seedFun = 1, seedSPOT = 1, verbosity = 0, designControl=list(size=n) ), kerasConf = kerasConf, data = mnist ) save(res, file = paste0("resKerasMnist02Default", as.numeric(Sys.time()), ".RData"))
KerasMnist03: same as 02, but training loss passed to optimizer
install.packages(c( "benchmarkme", "callr", "emoa", "ggsci", "jsonlite", "keras", "magrittr", "mlr", "plotly", "RColorBrewer", "reticulate", "rpart.plot", "sensitivity", "smoof", "tensorflow", "tfdatasets")) install.packages( "~/workspace/spotmisc/packages/SPOT_2.7.0.tar.gz", repos = NULL, type = "source" ) install.packages( "~/workspace/spotmisc/packages/SPOTMisc_1.6.0.tar.gz", repos = NULL, type = "source" ) library("SPOTMisc") packageVersion("SPOTMisc") library("SPOT") packageVersion("SPOT") mnist <- getMnistData() # "dropout" = x[1]: 0.1--0.5 # "dropoutfact" = x[2]: 0.85--0.95 # "units" = x[3]: 7-9 (128--512) # "unitsfact" = x[4]: 0.5--1.5 # "learning_rate" = x[5]: 1e-5--1e-2 # "epochs" = x[6]: 3-6 (10--50) # "batchsize" = x[7]: 6-9 (64--512) # "beta_1" = x[8]: 0.9--0.999 # "beta_2" = x[9]: 0.99--0.9999 # "layers" = x[10]: 1--5 # "epsilon" = x[11]: 1e-8--1e-8 lower <- c(0.1, 0.85, 7, 0.5, 1e-5, 3, 6, 0.9, 0.99, 1, 1e-8) upper <- c(0.5, 0.95, 9, 1.5, 1e-2, 6, 9, 0.999, 0.9999, 5, 1e-6) types <- rep("numeric", length(lower)) types[c(3, 6, 7)] <- "integer" f2 <- function(x) { 2 ^ x } transformFun <- rep("identity", length(lower)) transformFun[c(3, 6, 7)] <- "f2" kerasConf <- getKerasConf() kerasConf$verbose <- 0 kerasConf$resDummy <- FALSE kerasConf$returnValue <- "trainingLoss" Ninit <- 2 * length(lower) Rinit <- 2 replicates <- 2 res <- spot( x = NULL, fun = funKerasMnist, lower = lower, upper = upper, control = list( funEvals = 20 * length(lower), noise = TRUE, types = types, transformFun = transformFun, replicates = replicates, designControl = list(replicates = Rinit, size = Ninit), model = buildKriging, optimizer = optimLBFGSB, modelControl = list(target = "ei"), plots = FALSE, progress = TRUE, seedFun = 1, seedSPOT = 1 ), kerasConf = kerasConf, data = mnist ) save(res, file = paste0("resKerasMnist03", as.numeric(Sys.time()),".RData"))
KerasMnist07: same as 02, but test loss passed to optimizer
install.packages(c( "benchmarkme", "callr", "emoa", "ggsci", "jsonlite", "keras", "magrittr", "mlr", "plotly", "RColorBrewer", "reticulate", "rpart.plot", "sensitivity", "smoof", "tensorflow", "tfdatasets")) install.packages( "~/workspace/spotmisc/packages/SPOT_2.7.0.tar.gz", repos = NULL, type = "source" ) install.packages( "~/workspace/spotmisc/packages/SPOTMisc_1.6.0.tar.gz", repos = NULL, type = "source" ) library("SPOTMisc") packageVersion("SPOTMisc") library("SPOT") packageVersion("SPOT") mnist <- getMnistData() # "dropout" = x[1]: 0.1--0.5 # "dropoutfact" = x[2]: 0.85--0.95 # "units" = x[3]: 7-9 (128--512) # "unitsfact" = x[4]: 0.5--1.5 # "learning_rate" = x[5]: 1e-5--1e-2 # "epochs" = x[6]: 3-6 (10--50) # "batchsize" = x[7]: 6-9 (64--512) # "beta_1" = x[8]: 0.9--0.999 # "beta_2" = x[9]: 0.99--0.9999 # "layers" = x[10]: 1--5 # "epsilon" = x[11]: 1e-8--1e-8 lower <- c(0.1, 0.85, 7, 0.5, 1e-5, 3, 6, 0.9, 0.99, 1, 1e-8) upper <- c(0.5, 0.95, 9, 1.5, 1e-2, 6, 9, 0.999, 0.9999, 5, 1e-6) types <- rep("numeric", length(lower)) types[c(3, 6, 7)] <- "integer" f2 <- function(x) { 2 ^ x } transformFun <- rep("identity", length(lower)) transformFun[c(3, 6, 7)] <- "f2" kerasConf <- getKerasConf() kerasConf$verbose <- 0 kerasConf$resDummy <- FALSE kerasConf$returnValue <- "testLoss" Ninit <- 2 * length(lower) Rinit <- 2 replicates <- 2 res <- spot( x = NULL, fun = funKerasMnist, lower = lower, upper = upper, control = list( funEvals = 20 * length(lower), noise = TRUE, types = types, transformFun = transformFun, replicates = replicates, designControl = list(replicates = Rinit, size = Ninit), model = buildKriging, optimizer = optimLBFGSB, modelControl = list(target = "ei"), plots = FALSE, progress = TRUE, seedFun = 1, seedSPOT = 1 ), kerasConf = kerasConf, data = mnist ) save(res, file = paste0("resKerasMnist07", as.numeric(Sys.time()),".RData"))
KerasMnist08: same as 02, but negative validation accuracy passed to optimizer
install.packages(c( "benchmarkme", "callr", "emoa", "ggsci", "jsonlite", "keras", "magrittr", "mlr", "plotly", "RColorBrewer", "reticulate", "rpart.plot", "sensitivity", "smoof", "tensorflow", "tfdatasets")) install.packages( "~/workspace/spotmisc/packages/SPOT_2.7.0.tar.gz", repos = NULL, type = "source" ) install.packages( "~/workspace/spotmisc/packages/SPOTMisc_1.6.0.tar.gz", repos = NULL, type = "source" ) library("SPOTMisc") packageVersion("SPOTMisc") library("SPOT") packageVersion("SPOT") mnist <- getMnistData() # "dropout" = x[1]: 0.1--0.5 # "dropoutfact" = x[2]: 0.85--0.95 # "units" = x[3]: 7-9 (128--512) # "unitsfact" = x[4]: 0.5--1.5 # "learning_rate" = x[5]: 1e-5--1e-2 # "epochs" = x[6]: 3-6 (10--50) # "batchsize" = x[7]: 6-9 (64--512) # "beta_1" = x[8]: 0.9--0.999 # "beta_2" = x[9]: 0.99--0.9999 # "layers" = x[10]: 1--5 # "epsilon" = x[11]: 1e-8--1e-8 lower <- c(0.1, 0.85, 7, 0.5, 1e-5, 3, 6, 0.9, 0.99, 1, 1e-8) upper <- c(0.5, 0.95, 9, 1.5, 1e-2, 6, 9, 0.999, 0.9999, 5, 1e-6) types <- rep("numeric", length(lower)) types[c(3, 6, 7)] <- "integer" f2 <- function(x) { 2 ^ x } transformFun <- rep("identity", length(lower)) transformFun[c(3, 6, 7)] <- "f2" kerasConf <- getKerasConf() kerasConf$verbose <- 0 kerasConf$resDummy <- FALSE kerasConf$returnValue <- "negValidationAccuracy" Ninit <- 2 * length(lower) Rinit <- 2 replicates <- 2 res <- spot( x = NULL, fun = funKerasMnist, lower = lower, upper = upper, control = list( funEvals = 20 * length(lower), noise = TRUE, types = types, transformFun = transformFun, replicates = replicates, designControl = list(replicates = Rinit, size = Ninit), model = buildKriging, optimizer = optimLBFGSB, modelControl = list(target = "ei"), plots = FALSE, progress = TRUE, seedFun = 1, seedSPOT = 1 ), kerasConf = kerasConf, data = mnist ) save(res, file = paste0("resKerasMnist08", as.numeric(Sys.time()),".RData"))
KerasMnist09: same as 02, but negative training accuracy passed to optimizer
install.packages(c( "benchmarkme", "callr", "emoa", "ggsci", "jsonlite", "keras", "magrittr", "mlr", "plotly", "RColorBrewer", "reticulate", "rpart.plot", "sensitivity", "smoof", "tensorflow", "tfdatasets")) install.packages( "~/workspace/spotmisc/packages/SPOT_2.7.0.tar.gz", repos = NULL, type = "source" ) install.packages( "~/workspace/spotmisc/packages/SPOTMisc_1.6.0.tar.gz", repos = NULL, type = "source" ) library("SPOTMisc") packageVersion("SPOTMisc") library("SPOT") packageVersion("SPOT") mnist <- getMnistData() # "dropout" = x[1]: 0.1--0.5 # "dropoutfact" = x[2]: 0.85--0.95 # "units" = x[3]: 7-9 (128--512) # "unitsfact" = x[4]: 0.5--1.5 # "learning_rate" = x[5]: 1e-5--1e-2 # "epochs" = x[6]: 3-6 (10--50) # "batchsize" = x[7]: 6-9 (64--512) # "beta_1" = x[8]: 0.9--0.999 # "beta_2" = x[9]: 0.99--0.9999 # "layers" = x[10]: 1--5 # "epsilon" = x[11]: 1e-8--1e-8 lower <- c(0.1, 0.85, 7, 0.5, 1e-5, 3, 6, 0.9, 0.99, 1, 1e-8) upper <- c(0.5, 0.95, 9, 1.5, 1e-2, 6, 9, 0.999, 0.9999, 5, 1e-6) types <- rep("numeric", length(lower)) types[c(3, 6, 7)] <- "integer" f2 <- function(x) { 2 ^ x } transformFun <- rep("identity", length(lower)) transformFun[c(3, 6, 7)] <- "f2" kerasConf <- getKerasConf() kerasConf$verbose <- 0 kerasConf$resDummy <- FALSE kerasConf$returnValue <- "negTrainingAccuracy" Ninit <- 2 * length(lower) Rinit <- 2 replicates <- 2 res <- spot( x = NULL, fun = funKerasMnist, lower = lower, upper = upper, control = list( funEvals = 20 * length(lower), noise = TRUE, types = types, transformFun = transformFun, replicates = replicates, designControl = list(replicates = Rinit, size = Ninit), model = buildKriging, optimizer = optimLBFGSB, modelControl = list(target = "ei"), plots = FALSE, progress = TRUE, seedFun = 1, seedSPOT = 1 ), kerasConf = kerasConf, data = mnist ) save(res, file = paste0("resKerasMnist09", as.numeric(Sys.time()),".RData"))
KerasMnist10: same as 02, but negative test accuracy passed to optimizer
install.packages(c( "benchmarkme", "callr", "emoa", "ggsci", "jsonlite", "keras", "magrittr", "mlr", "plotly", "RColorBrewer", "reticulate", "rpart.plot", "sensitivity", "smoof", "tensorflow", "tfdatasets")) install.packages( "~/workspace/spotmisc/packages/SPOT_2.7.0.tar.gz", repos = NULL, type = "source" ) install.packages( "~/workspace/spotmisc/packages/SPOTMisc_1.6.0.tar.gz", repos = NULL, type = "source" ) library("SPOTMisc") packageVersion("SPOTMisc") library("SPOT") packageVersion("SPOT") mnist <- getMnistData() # "dropout" = x[1]: 0.1--0.5 # "dropoutfact" = x[2]: 0.85--0.95 # "units" = x[3]: 7-9 (128--512) # "unitsfact" = x[4]: 0.5--1.5 # "learning_rate" = x[5]: 1e-5--1e-2 # "epochs" = x[6]: 3-6 (10--50) # "batchsize" = x[7]: 6-9 (64--512) # "beta_1" = x[8]: 0.9--0.999 # "beta_2" = x[9]: 0.99--0.9999 # "layers" = x[10]: 1--5 # "epsilon" = x[11]: 1e-8--1e-8 lower <- c(0.1, 0.85, 7, 0.5, 1e-5, 3, 6, 0.9, 0.99, 1, 1e-8) upper <- c(0.5, 0.95, 9, 1.5, 1e-2, 6, 9, 0.999, 0.9999, 5, 1e-6) types <- rep("numeric", length(lower)) types[c(3, 6, 7)] <- "integer" f2 <- function(x) { 2 ^ x } transformFun <- rep("identity", length(lower)) transformFun[c(3, 6, 7)] <- "f2" kerasConf <- getKerasConf() kerasConf$verbose <- 0 kerasConf$resDummy <- FALSE kerasConf$returnValue <- "negTestAccuracy" Ninit <- 2 * length(lower) Rinit <- 2 replicates <- 2 res <- spot( x = NULL, fun = funKerasMnist, lower = lower, upper = upper, control = list( funEvals = 20 * length(lower), noise = TRUE, types = types, transformFun = transformFun, replicates = replicates, designControl = list(replicates = Rinit, size = Ninit), model = buildKriging, optimizer = optimLBFGSB, modelControl = list(target = "ei"), plots = FALSE, progress = TRUE, seedFun = 1, seedSPOT = 1 ), kerasConf = kerasConf, data = mnist ) save(res, file = paste0("resKerasMnist10", as.numeric(Sys.time()),".RData"))
Transfer Learning
resKerasTransferLearning04
library("SPOTMisc") library("SPOT") # Hyperparameters: # "dropout" = x[1], # "learning_rate" = x[2], # "epochs" = x[3], # "beta_1" = x[4], # "beta_2" = x[5], # "epsilon" = x[6], # "optimizer" = x[7] lower <- c(1e-6, 1e-6, 2, 0.8, 0.8, 1e-9, 1) upper <- c(0.2, 1e-2, 5, 0.99, 0.9999, 1e-3, 2) types <- c("numeric", "numeric", "integer", "numeric", "numeric", "numeric", "factor") kerasConf <- getKerasConf() kerasConf$verbose <- 1 kerasConf$resDummy <- FALSE kerasConf$naDummy <- 1234567890 res <- spot(x = NULL, fun = funKerasTransferLearning, lower = lower, upper = upper, control = list(funEvals=50, model=buildKriging, noise = TRUE, types = types, optimizer=optimDE, plots = FALSE, progress = TRUE, seedFun = 1, seedSPOT = 1, verbosity = 1), kerasConf = kerasConf ) save(res, file = paste0("resKerasTransferLearning04", as.numeric(Sys.time()),".RData"))
resKerasTransferLearning05
library("SPOTMisc") library("SPOT") kerasConf <- getKerasConf() # Hyperparameters: # "dropout" = x[1], # "learning_rate" = x[2], # "epochs" = x[3], # "beta_1" = x[4], # "beta_2" = x[5], # "epsilon" = x[6], # "optimizer" = x[7] lower <- c(1e-6, 1e-6, 2, 0.8, 0.8, 1e-9, 1) upper <- c(0.2, 1e-2, 5, 0.99, 0.9999, 1e-3, 2) types <- c("numeric", "numeric", "integer", "numeric", "numeric", "numeric", "factor") ### Example: spot call with extended verbosity and negative val accuracy: kerasConf$verbose <- 0 kerasConf$returnValue <- "negValidationAccuracy" res <- spot(x = NULL, fun = funKerasTransferLearning, lower = lower, upper = upper, control = list(funEvals=50, model=buildKriging, noise = TRUE, types = types, optimizer=optimDE, plots = FALSE, progress = TRUE, seedFun = 1, seedSPOT = 1, verbosity = 1), kerasConf = kerasConf ) save(res, file = paste0("resKerasTransferLearning05", as.numeric(Sys.time()),".RData"))
resKerasTransferLearning06
- Evaluation of the default hyperparameter setting (baseline)
library("SPOTMisc") library("SPOT") kerasConf <- getKerasConf() # Hyperparameters: # "dropout" = x[1], # "learning_rate" = x[2], # "epochs" = x[3], # "beta_1" = x[4], # "beta_2" = x[5], # "epsilon" = x[6], # "optimizer" = x[7] replicates <- 1 lower <- c(0.2, 1e-5, 5, 0.9, 0.999, 1e-7, 1) upper <- c(0.2, 1e-5, 5, 0.9, 0.999, 1e-7, 1) types <- c("numeric", "numeric", "integer", "numeric", "numeric", "numeric", "factor") ### Example: spot call to evaluate the default design kerasConf$verbose <- 0 kerasConf$returnValue <- "validationLoss" kerasConf$naDummy <- 1234567890 res <- spot(x = NULL, fun = funKerasTransferLearning, lower = lower, upper = upper, control = list(funEvals=10, model=buildKriging, noise = TRUE, types = types, optimizer = optimDE, plots = FALSE, progress = TRUE, designControl=list(replicates=replicates), seedFun = 1, seedSPOT = 1, verbosity = 1), kerasConf = kerasConf ) save(res, file = paste0("resKerasTransferLearning06", as.numeric(Sys.time()),".RData"))
kerasMnist11: 02 with Multi Start
install.packages(c( "benchmarkme", "callr", "emoa", "ggsci", "jsonlite", "keras", "magrittr", "mlr", "plotly", "RColorBrewer", "reticulate", "rpart.plot", "sensitivity", "smoof", "tensorflow", "tfdatasets")) install.packages( "~/workspace/spotmisc/packages/SPOT_2.8.4.tar.gz", repos = NULL, type = "source" ) install.packages( "~/workspace/spotmisc/packages/SPOTMisc_1.8.0.tar.gz", repos = NULL, type = "source" ) library("SPOTMisc") packageVersion("SPOTMisc") library("SPOT") packageVersion("SPOT") mnist <- getMnistData() # "dropout" = x[1]: 0.1--0.5 # "dropoutfact" = x[2]: 0.85--0.95 # "units" = x[3]: 7-9 (128--512) # "unitsfact" = x[4]: 0.5--1.5 # "learning_rate" = x[5]: 1e-5--1e-2 # "epochs" = x[6]: 3-6 (10--50) # "batchsize" = x[7]: 6-9 (64--512) # "beta_1" = x[8]: 0.9--0.999 # "beta_2" = x[9]: 0.99--0.9999 # "layers" = x[10]: 1--5 # "epsilon" = x[11]: 1e-8--1e-8 lower <- c(0.1, 0.85, 7, 0.5, 1e-5, 3, 6, 0.9, 0.99, 1, 1e-8) upper <- c(0.5, 0.95, 9, 1.5, 1e-2, 6, 9, 0.999, 0.9999, 5, 1e-6) types <- rep("numeric", length(lower)) types[c(3, 6, 7)] <- "integer" f2 <- function(x) { 2 ^ x } transformFun <- rep("identity", length(lower)) transformFun[c(3, 6, 7)] <- "f2" kerasConf <- getKerasConf() kerasConf$verbose <- 0 kerasConf$resDummy <- TRUE Ninit <- 2 * length(lower) Rinit <- 2 replicates <- 2 res <- spot( x = NULL, fun = funKerasMnist, lower = lower, upper = upper, control = list( funEvals = 20 * length(lower), multiStart = 2, noise = TRUE, types = types, transformFun = transformFun, replicates = replicates, designControl = list(replicates = Rinit, size = Ninit), model = buildKriging, optimizer = optimDE, modelControl = list(target = "ei"), plots = FALSE, progress = TRUE, seedFun = 1, seedSPOT = 1, verbosity=0 ), kerasConf = kerasConf, data = mnist ) save(res, file = paste0("resKerasMnist11", as.numeric(Sys.time()), ".RData"))
Census Data
# install.packages("~/workspace/book/packages/SPOTMisc_1.8.2.tar.gz", repos = NULL, type = "source") # install.packages("~/workspace/book/packages/SPOT_2.8.4.tar.gz", repos = NULL, type = "source") library("SPOTMisc") if (packageVersion("SPOTMisc") < "1.8.0") stop("Please update 'SPOT'") library("SPOT") if (packageVersion("SPOT") < "2.8.4") stop("Please update 'SPOT'") # "dropout" = x[1]: 0.1--0.5 # "dropoutfact" = x[2]: 1-1 # "units" = x[3]: 1-1 # "unitsfact" = x[4]: 1-1 # "learning_rate" = x[5]: 1e-5--1e-2 # "epochs" = x[6]: 3-6 # "beta_1" = x[7]: 0.9--0.999 # "beta_2" = x[8]: 0.99--0.9999 # "layers" = x[9]: 1-1 # "epsilon" = x[10]: 1e-8--1e-6 lower <- c(0.1, 1, 1, 1, 1e-5, 3, 0.9, 0.99, 1, 1e-8) upper <- c(0.5, 1, 1, 1, 1e-2, 6, 0.999, 0.9999, 1, 1e-6) types <- rep("numeric", length(lower)) types[c(3, 4, 6, 9)] <- "integer" f2 <- function(x) { 2 ^ x } transformFun <- rep("identity", length(lower)) transformFun[c(3, 4, 6, 9)] <- "f2" kerasConf <- getKerasConf() kerasConf$verbose <- 0 kerasConf$resDummy <- FALSE Ninit <- 2 * length(lower) Rinit <- 2 replicates <- 2 task.type <- "classif" nobs <- 1e4 # max: 229285 nfactors = "high" nnumericals <- "high" cardinality = "high" data.seed <- 1 tuner.seed <- 1 timebudget <- 3600 timeout <- timebudget/20 data <- getCensusTrainValTestData(task.type=task.type, nobs=nobs, nfactors = nfactors, nnumericals = nnumericals, cardinality = cardinality, data.seed =1) specCensusPrep <- censusDataPrep(data=data, batch_size = 32) res <- spot( x = NULL, fun = funKerasCensus, lower = lower, upper = upper, control = list( funEvals = 20 * length(lower), multiStart = 2, noise = TRUE, types = types, transformFun = transformFun, replicates = replicates, designControl = list(replicates = Rinit, size = Ninit), model = buildKriging, optimizer = optimDE, modelControl = list(target = "y"), plots = FALSE, progress = TRUE, seedFun = 1, seedSPOT = 1, verbosity=0 ), kerasConf = kerasConf, specCensusPrep =specCensusPrep ) save(res, file = paste0("rescs3Census01", as.numeric(Sys.time()), ".RData"))
Try the SPOTMisc package in your browser
Any scripts or data that you put into this service are public.
R Package Documentation
Browse R Packages
We want your feedback!
Note that we can't provide technical support on individual packages. You should contact the package authors for that.
Embedding an R snippet on your website
Add the following code to your website.
For more information on customizing the embed code, read Embedding Snippets.