R/create_spark_connection.R

In hktosun/gozentosun2021: Replication of Gozen & Tosun (2021)

#' Create Spark connection
#'
#' Create Spark connection with pre-set configuration.
#'
#' @param JAVA_HOME Path to Java
#' @export

create_spark_connection <- function(JAVA_HOME = Sys.getenv('JAVA_HOME')){

	config <- sparklyr::spark_config()
	config$`sparklyr.shell.driver-memory` <- "4G"
	config$`sparklyr.shell.executor-memory` <- "4G"
	config$spark.driver.maxResultSize <- "4G"
	config$spark.yarn.executor.memoryOverhead <- "1g"

	sc <- sparklyr::spark_connect(master = "local", version = "2.3", config = config)
	sc
}