R/train_cpc.R

In GenomeNet/deepG: Deep Learning for Genome Sequence Data

#' @title Train CPC inspired model
#'   
#' @description
#' Train a CPC (Oord et al.) inspired neural network on genomic data.
#' 
#' @inheritParams generator_fasta_lm
#' @inheritParams generator_fasta_label_folder
#' @inheritParams generator_fasta_label_header_csv
#' @inheritParams train_model
#' @param train_type Either `"cpc"`, `"Self-GenomeNet"`. 
#' @param encoder A keras encoder for the cpc function. 
#' @param context A keras context model for the cpc function.
#' @param path Path to training data. If \code{train_type} is \code{label_folder}, should be a vector or list
#' where each entry corresponds to a class (list elements can be directories and/or individual files). If \code{train_type} is not \code{label_folder}, 
#' can be a single directory or file or a list of directories and/or files.
#' @param path_val Path to validation data. See `path` argument for details.
#' @param path_checkpoint Path to checkpoints folder or `NULL`. If `NULL`, checkpoints don't get stored.
#' @param path_tensorboard Path to tensorboard directory or `NULL`. If `NULL`, training not tracked on tensorboard.
#' @param train_val_ratio For generator defines the fraction of batches that will be used for validation (compared to size of training data), i.e. one validation iteration
#' processes \code{batch_size} \eqn{*} \code{steps_per_epoch} \eqn{*} \code{train_val_ratio} samples. If you use dataset instead of generator and \code{dataset_val} is `NULL`, splits \code{dataset}
#' into train/validation data.
#' @param run_name Name of the run. Name will be used to identify output from callbacks.
#' @param batch_size Number of samples used for one network update.
#' @param epochs Number of iterations.
#' @param steps_per_epoch Number of training batches per epoch.
#' @param shuffle_file_order Boolean, whether to go through files sequentially or shuffle beforehand.
#' @param initial_epoch Epoch at which to start training. Note that network
#' will run for (\code{epochs} - \code{initial_epochs}) rounds and not \code{epochs} rounds.
#' @param seed Sets seed for reproducible results.
#' @param file_limit Integer or `NULL`. If integer, use only specified number of randomly sampled files for training. Ignored if greater than number of files in \code{path}.
#' @param patchlen The length of a patch when splitting the input sequence.
#' @param nopatches The number of patches when splitting the input sequence. 
#' @param step Frequency of sampling steps.
#' @param stride The overlap between two patches when splitting the input sequence.
#' @param pretrained_model A pretrained keras model, for which training will be continued
#' @param learningrate A Tensor, floating point value. If a schedule is defines, this value gives the initial learning rate. Defaults to 0.001.
#' @param learningrate_schedule A schedule for a non-constant learning rate over the training. Either "cosine_annealing", "step_decay", or "exp_decay".
#' @param k Value of k for sparse top k categorical accuracy. Defaults to 5.
#' @param stepsmin In CPC, a patch is predicted given another patch. stepsmin defines how many patches between these two should be ignored during prediction.
#' @param stepsmax The maximum distance between the predicted patch and the given patch.
#' @param emb_scale Scales the impact of a patches context.
#' @examplesIf reticulate::py_module_available("tensorflow")
#' 
#' #create dummy data
#' path_train_1 <- tempfile()
#' path_train_2 <- tempfile()
#' path_val_1 <- tempfile()
#' path_val_2 <- tempfile()
#' 
#' for (current_path in c(path_train_1, path_train_2,
#'                        path_val_1, path_val_2)) {
#'   dir.create(current_path)
#'   deepG::create_dummy_data(file_path = current_path,
#'                            num_files = 3,
#'                            seq_length = 10,
#'                            num_seq = 5,
#'                            vocabulary = c("a", "c", "g", "t"))
#' }
#' 
#' # create model
#' encoder <- function(maxlen = NULL,
#'                     patchlen = NULL,
#'                     nopatches = NULL,
#'                     eval = FALSE) {
#'   if (is.null(nopatches)) {
#'     nopatches <- nopatchescalc(patchlen, maxlen, patchlen * 0.4)
#'   }
#'   inp <- keras::layer_input(shape = c(maxlen, 4))
#'   stridelen <- as.integer(0.4 * patchlen)
#'   createpatches <- inp %>%
#'     keras::layer_reshape(list(maxlen, 4L, 1L), name = "prep_reshape1", dtype = "float32") %>%
#'     tensorflow::tf$image$extract_patches(
#'       sizes = list(1L, patchlen, 4L, 1L),
#'       strides = list(1L, stridelen, 4L, 1L),
#'       rates = list(1L, 1L, 1L, 1L),
#'       padding = "VALID",
#'       name = "prep_patches"
#'     ) %>%
#'     keras::layer_reshape(list(nopatches, patchlen, 4L),
#'                          name = "prep_reshape2") %>%
#'     tensorflow::tf$reshape(list(-1L, patchlen, 4L),
#'                            name = "prep_reshape3")
#' 
#'   danQ <- createpatches %>%
#'     keras::layer_conv_1d(
#'       input_shape = c(maxlen, 4L),
#'       filters = 320L,
#'       kernel_size = 26L,
#'       activation = "relu"
#'     ) %>%
#'     keras::layer_max_pooling_1d(pool_size = 13L, strides = 13L) %>%
#'     keras::layer_dropout(0.2) %>%
#'     keras::layer_lstm(units = 320, return_sequences = TRUE) %>%
#'     keras::layer_dropout(0.5) %>%
#'     keras::layer_flatten() %>%
#'     keras::layer_dense(925, activation = "relu")
#'   patchesback <- danQ %>%
#'     tensorflow::tf$reshape(list(-1L, tensorflow::tf$cast(nopatches, tensorflow::tf$int16), 925L))
#'   keras::keras_model(inp, patchesback)
#' }
#' 
#' context <- function(latents) {
#'   cres <- latents
#'   cres_dim = cres$shape
#'   predictions <-
#'     cres %>%
#'     keras::layer_lstm(
#'       return_sequences = TRUE,
#'       units = 256,  # WAS: 2048,
#'       name = paste("context_LSTM_1",
#'                    sep = ""),
#'       activation = "relu"
#'     )
#'   return(predictions)
#' }
#' 
#' # train model
#' temp_dir <- tempdir()
#' hist <- train_model_cpc(train_type = "CPC",
#'                         ### cpc functions ###
#'                         encoder = encoder,
#'                         context = context,
#'                         #### Generator settings ####
#'                         path_checkpoint = temp_dir,
#'                         path = c(path_train_1, path_train_2),
#'                         path_val = c(path_val_1, path_val_2),
#'                         run_name = "TEST",
#'                         batch_size = 8,
#'                         epochs = 3,
#'                         steps_per_epoch = 6,
#'                         patchlen = 100,
#'                         nopatches = 8)
#'                 
#'  
#' @returns A list of training metrics.  
#' @export
train_model_cpc <-
  function(train_type = "CPC",
           ### cpc functions ###
           encoder = NULL,
           context = NULL,
           #### Generator settings ####
           path,
           path_val = NULL,
           path_checkpoint = NULL,
           path_tensorboard = NULL,
           train_val_ratio = 0.2,
           run_name,
           
           batch_size = 32,
           epochs = 100,
           steps_per_epoch = 2000,
           shuffle_file_order = FALSE,
           initial_epoch = 1,
           seed = 1234,
           
           path_file_log = TRUE,
           train_val_split_csv = NULL,
           file_limit = NULL,
           proportion_per_seq = NULL,
           max_samples = NULL,
           maxlen = NULL,
           
           patchlen = NULL,
           nopatches = NULL,
           step = NULL,
           file_filter = NULL,
           stride = 0.4,
           pretrained_model = NULL,
           learningrate = 0.001,
           learningrate_schedule = NULL,
           k = 5,
           stepsmin = 2,
           stepsmax = 3,
           emb_scale = 0.1) {
    
    # Stride is default 0.4 x patchlen FOR NOW
    stride <- 0.4
    
    patchlen <- as.integer(patchlen)
    
    ########################################################################################################
    ############################### Warning messages if wrong initialization ###############################
    ########################################################################################################
    
    ####~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Model specification ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~####
    ## Three options:
    ## 1. Define Maxlen and Patchlen
    ## 2. Define Number of patches and Patchlen
    ## ---> in both cases the respectively missing value will be calculated
    ## 3. Pretrained model is giving specs
    ## error if none of those is fulfilled
    
    if (is.null(pretrained_model)) {
      ## If no pretrained model, patchlen has to be defined
      if (is.null(patchlen)) {
        stop("Please define patchlen")
      }
      ## Either maxlen or number of patches is needed
      if (is.null(maxlen) & is.null(nopatches)) {
        stop("Please define either maxlen or nopatches")
        ## the respectively missing value will be calculated
      } else if (is.null(maxlen) & !is.null(nopatches)) {
        maxlen <- (nopatches - 1) * (stride * patchlen) + patchlen
      } else if (!is.null(maxlen) & is.null(nopatches)) {
        nopatches <-
          as.integer((maxlen - patchlen) / (stride * patchlen) + 1)
      }
      ## if step is not defined, we do not use overlapping sequences
      if (is.null(step)) {
        step = maxlen
      }
    } else if (!is.null(pretrained_model)) {
      specs <-
        readRDS(paste(
          sub("/[^/]+$", "", pretrained_model),
          "modelspecs.rds",
          sep = "/"
        ))
      patchlen          <- specs$patchlen
      maxlen            <- specs$maxlen
      nopatches         <- specs$nopatches
      stride            <- specs$stride
      step              <- specs$step
      k                 <- specs$k
      emb_scale         <- specs$emb_scale
    }
    
    
    ####~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Learning rate schedule ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~####
    ## If learning_rate schedule is wanted, all necessary parameters must be given
    LRstop(learningrate_schedule)
    ########################################################################################################
    #################################### Preparation: Data, paths metrics ##################################
    ########################################################################################################
    
    ####~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Path definition ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~####
    runname <-
      paste0(run_name , format(Sys.time(), "_%y%m%d_%H%M%S"))
    
    ## Create folder for model
    if (!is.null(path_checkpoint)) {
      dir.create(paste(path_checkpoint, runname, sep = "/"))
      dir <- paste(path_checkpoint, runname, sep = "/")
      ## Create folder for filelog
      path_file_log <-
        paste(path_checkpoint, runname, "filelog.csv", sep = "/")
    } else {
      path_file_log <- NULL
    }
    
    GenConfig <-
      GenParams(maxlen, batch_size, step, proportion_per_seq, max_samples)
    GenTConfig <-
      GenTParams(path, shuffle_file_order, path_file_log, seed)
    GenVConfig <- GenVParams(path_val, shuffle_file_order)
    
    # train train_val_ratio via csv file
    if (!is.null(train_val_split_csv)) {
      if (is.null(path_val)) {
        path_val <- path
      } else {
        if (!all(unlist(path_val) %in% unlist(path))) {
          warning("Train/validation split done via file in train_val_split_csv. Only using files from path argument.")
        }
        path_val <- path
      }
      
      train_val_file <- utils::read.csv2(train_val_split_csv, header = TRUE, stringsAsFactors = FALSE)
      if (dim(train_val_file)[2] == 1) {
        train_val_file <- utils::read.csv(train_val_split_csv, header = TRUE, stringsAsFactors = FALSE)
      }
      train_val_file <- dplyr::distinct(train_val_file)
      
      if (!all(c("file", "type") %in% names(train_val_file))) {
        stop("Column names of train_val_split_csv file must be 'file' and 'type'")
      }
      
      if (length(train_val_file$file) != length(unique(train_val_file$file))) {
        stop("In train_val_split_csv all entires in 'file' column must be unique")
      }
      
      file_filter <- list()
      file_filter[[1]] <- train_val_file %>% dplyr::filter(type == "train")
      file_filter[[1]] <- as.character(file_filter[[1]]$file)
      file_filter[[2]] <- train_val_file %>% dplyr::filter(type == "val" | type == "validation")
      file_filter[[2]] <- as.character(file_filter[[2]]$file)
    }
    
    
    ####~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ File count ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~####
    if (is.null(file_filter) && is.null(train_val_split_csv)) {
      if (is.null(file_limit)) {
        if (is.list(path)) {
          num_files <- 0
          for (i in seq_along(path)) {
            num_files <- num_files + length(list.files(path[[i]]))
          }
        } else {
          num_files <- length(list.files(path))
        }
      } else {
        num_files <- file_limit * length(path)
      }
    } else {
      num_files <- length(file_filter[1])
    }
    
    ####~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Creation of generators ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~####
    message(format(Sys.time(), "%F %R"), ": Preparing the data\n")
    ####~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Training Generator ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~####
    fastrain <-
      do.call(generator_fasta_lm,
              c(GenConfig, GenTConfig, file_filter = file_filter[1]))
    
    
    ####~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Validation Generator ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~####
    fasval <-
      do.call(
        generator_fasta_lm,
        c(
          GenConfig,
          GenVConfig,
          seed = seed,
          file_filter = file_filter[2]
        )
      )
    ####~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Creation of metrics ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~####
    message(format(Sys.time(), "%F %R"), ": Preparing the metrics\n")
    train_loss <- tensorflow::tf$keras$metrics$Mean(name = 'train_loss')
    val_loss <- tensorflow::tf$keras$metrics$Mean(name = 'val_loss')
    train_acc <- tensorflow::tf$keras$metrics$Mean(name = 'train_acc')
    val_acc <- tensorflow::tf$keras$metrics$Mean(name = 'val_acc')
    
    ########################################################################################################
    ###################################### History object preparation ######################################
    ########################################################################################################
    
    history <- list(
      params = list(
        batch_size = batch_size,
        epochs = 0,
        steps = steps_per_epoch,
        samples = steps_per_epoch * batch_size,
        verbose = 1,
        do_validation = TRUE,
        metrics = c("loss", "accuracy", "val_loss", "val_accuracy")
      ),
      metrics = list(
        loss = c(),
        accuracy = c(),
        val_loss = c(),
        val_accuracy = c()
      )
    )
    
    eploss <- list()
    epacc <- list()
    
    ####~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Reformat to S3 object ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~####
    class(history) <- "keras_training_history"
    
    ########################################################################################################
    ############################################ Model creation ############################################
    ########################################################################################################
    if (is.null(pretrained_model)) {
      ####~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Unsupervised Build from scratch ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~####
      message(format(Sys.time(), "%F %R"), ": Creating the model\n")
      ## Build encoder
      enc <-
        encoder(maxlen = maxlen,
                patchlen = patchlen,
                nopatches = nopatches)
      
      ## Build model
      model <-
        keras::keras_model(
          enc$input,
          cpcloss(
            enc$output,
            context,
            batch_size = batch_size,
            steps_to_ignore = stepsmin,
            steps_to_predict = stepsmax,
            train_type = train_type,
            k = k,
            emb_scale = emb_scale
          )
        )
      
      ## Build optimizer
      optimizer <- # keras::optimizer_adam(
        tensorflow::tf$keras$optimizers$legacy$Adam(
          learning_rate = learningrate,
          beta_1 = 0.8,
          epsilon = 10 ^ -8,
          decay = 0.999,
          clipnorm = 0.01
        )
      ####~~~~~~~~~~~~~~~~~~~~~~~~~~ Unsupervised Read if pretrained model given ~~~~~~~~~~~~~~~~~~~~~~~~~####
      
    } else {
      message(format(Sys.time(), "%F %R"), ": Loading the trained model.\n")
      ## Read model
      model <- keras::load_model_hdf5(pretrained_model, compile = FALSE)
      optimizer <- ReadOpt(pretrained_model)
      optimizer$learning_rate$assign(learningrate)
    }
    
    ####~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Saving necessary model objects ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~####
    ## optimizer configuration
    
    if (!is.null(path_checkpoint)) {
      saveRDS(optimizer$get_config(),
              paste(dir, "optconfig.rds", sep = "/"))
      ## model parameters
      saveRDS(
        list(
          maxlen = maxlen,
          patchlen = patchlen,
          stride = stride,
          nopatches = nopatches,
          step = step,
          batch_size = batch_size,
          epochs = epochs,
          steps_per_epoch = steps_per_epoch,
          train_val_ratio = train_val_ratio,
          max_samples = max_samples,
          k = k,
          emb_scale = emb_scale,
          learningrate = learningrate
        ),
        paste(dir, "modelspecs.rds", sep = "/")
      )
    }
    ########################################################################################################
    ######################################## Tensorboard connection ########################################
    ########################################################################################################
    
    if (!is.null(path_tensorboard)) {
      ####~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Initialize Tensorboard writers ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~####
      logdir <- path_tensorboard
      writertrain <-
        tensorflow::tf$summary$create_file_writer(file.path(logdir, runname, "/train"))
      writerval <-
        tensorflow::tf$summary$create_file_writer(file.path(logdir, runname, "/validation"))
      
      ####~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Write parameters to Tensorboard ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~####
      tftext <-
        lapply(as.list(match.call())[-1][-c(1, 2)], function(x)
          ifelse(all(nchar(deparse(
            eval(x)
          )) < 20) && !is.null(eval(x)), eval(x), deparse(x)))
      
      with(writertrain$as_default(), {
        tensorflow::tf$summary$text("Specification",
                                    paste(
                                      names(tftext),
                                      tftext,
                                      sep = " = ",
                                      collapse = "  \n"
                                    ),
                                    step = 0L)
      })
    }
    
    ########################################################################################################
    ######################################## Training loop function ########################################
    ########################################################################################################
    
    train_val_loop <-
      function(batches = steps_per_epoch, epoch, train_val_ratio) {
        ####~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Start of loop ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~####
        for (i in c("train", "val")) {
          if (i == "val") {
            ## Calculate steps for validation
            batches <- ceiling(batches * train_val_ratio)
          }
          
          for (b in seq(batches)) {
            if (i == "train") {
              ####~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Training step ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~####
              ## If Learning rate schedule specified, calculate learning_rate for current epoch
              if (!is.null(learningrate_schedule)) {
                optimizer$learning_rate$assign(getEpochLR(learningrate_schedule, epoch))
              }
              ####~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Optimization step ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~####
              
              #with(tensorflow::tf$GradientTape() %as% tape, {
              with(reticulate::`%as%`(tensorflow::tf$GradientTape(), tape), {
                
                out <-
                  modelstep(fastrain(),
                            model,
                            train_type,
                            TRUE)
                l <- out[1]
                acc <- out[2]
              })
              
              gradients <-
                tape$gradient(l, model$trainable_variables)
              optimizer$apply_gradients(purrr::transpose(list(
                gradients, model$trainable_variables
              )))
              train_loss(l)
              train_acc(acc)
              
            } else {
              ####~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Validation step ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~####
              out <-
                modelstep(fasval(),
                          model,
                          train_type,
                          FALSE)
              
              l <- out[1]
              acc <- out[2]
              val_loss(l)
              val_acc(acc)
              
            }
            
            ## Print status of epoch
            if (b %in% seq(0, batches, by = batches / 10)) {
              message("-")
            }
          }
          
          ####~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ End of Epoch ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~####
          if (i == "train") {
            ## Training step
            # Write epoch result metrics value to tensorboard
            if (!is.null(path_tensorboard)) {
              TB_loss_acc(writertrain, train_loss, train_acc, epoch)
              with(writertrain$as_default(), {
                tensorflow::tf$summary$scalar('epoch_lr',
                                              optimizer$learning_rate,
                                              step = tensorflow::tf$cast(epoch, "int64"))
                tensorflow::tf$summary$scalar(
                  'training files seen',
                  nrow(
                    readr::read_csv(
                      path_file_log,
                      col_names = FALSE,
                      col_types = readr::cols()
                    )
                  ) / num_files,
                  step = tensorflow::tf$cast(epoch, "int64")
                )
              })
            }
            # Print epoch result metric values to console
            tensorflow::tf$print(" Train Loss",
                                 train_loss$result(),
                                 ", Train Acc",
                                 train_acc$result())
            
            # Save epoch result metric values to history object
            history$params$epochs <- epoch
            history$metrics$loss[epoch] <-
              as.double(train_loss$result())
            history$metrics$accuracy[epoch]  <-
              as.double(train_acc$result())
            
            # Reset states
            train_loss$reset_states()
            train_acc$reset_states()
            
          } else {
            ## Validation step
            # Write epoch result metrics value to tensorboard
            if (!is.null(path_tensorboard)) {
              TB_loss_acc(writerval, val_loss, val_acc, epoch)
            }
            
            # Print epoch result metric values to console
            tensorflow::tf$print(" Validation Loss",
                                 val_loss$result(),
                                 ", Validation Acc",
                                 val_acc$result())
            
            # save results for best model saving condition
            if (b == max(seq(batches))) {
              eploss[[epoch]] <- as.double(val_loss$result())
              epacc[[epoch]] <-
                as.double(val_acc$result())
            }
            
            # Save epoch result metric values to history object
            history$metrics$val_loss[epoch] <-
              as.double(val_loss$result())
            history$metrics$val_accuracy[epoch]  <-
              as.double(val_acc$result())
            
            # Reset states
            val_loss$reset_states()
            val_acc$reset_states()
          }
        }
        return(list(history,eploss,epacc))
      }
    
    ########################################################################################################
    ############################################# Training run #############################################
    ########################################################################################################
    
    
    message(format(Sys.time(), "%F %R"), ": Starting Training\n")
    
    ## Training loop
    for (i in seq(initial_epoch, (epochs + initial_epoch - 1))) {
      message(format(Sys.time(), "%F %R"), ": EPOCH ", i, " \n")
      
      ## Epoch loop
      out <- train_val_loop(epoch = i, train_val_ratio = train_val_ratio)
      history <- out[[1]]
      eploss <- out[[2]]
      epacc <- out[[3]]
      ## Save checkpoints
      # best model (smallest loss)
      if (eploss[[i]] == min(unlist(eploss))) {
        savechecks("best", runname, model, optimizer, history, path_checkpoint)
      }
      # backup model every 10 epochs
      if (i %% 2 == 0) {
        savechecks("backup", runname, model, optimizer, history, path_checkpoint)
      }
    }
    
    ########################################################################################################
    ############################################# Final saves ##############################################
    ########################################################################################################
    
    savechecks(cp = "FINAL", runname, model, optimizer, history, path_checkpoint)
    if (!is.null(path_tensorboard)) {
      writegraph <-
        tensorflow::tf$keras$callbacks$TensorBoard(file.path(logdir, runname))
      writegraph$set_model(model)
    }
  }