publications/LatinR2019/Presentation/Scripts/03_fit_model_w_output.R

In r-tensorflow/autokeras: R Interface to 'AutoKeras'

if (Sys.info()[["user"]] == "jcrodriguez") {
  setwd("~/mytmp/emoji_ds/")
}

library("autokeras")
library("keras")

## load data

set.seed(8818)
labels <- readRDS("labels.rds")
imgs_arr <- readRDS("imgs_arr.rds")

test_idxs <- sample(seq_len(nrow(labels)), round(nrow(imgs_arr) * .2))
x_train <- imgs_arr[-test_idxs, , , ]
y_train <- labels[-test_idxs, "Category"]
x_test <- imgs_arr[test_idxs, , , ]
y_test <- labels[test_idxs, "Category"]

rm(list = c("labels", "imgs_arr"))

## fit autokeras model

# Create an image classifier, and train different models
clf <- model_image_classifier(verbose = TRUE, augment = FALSE) %>%
  fit(x_train, y_train, time_limit = 1 * 60 * 60)

{
  # Saving Directory: /tmp/autokeras_54RG1M
  # Preprocessing the images.
  # Preprocessing finished.
  #
  # Initializing search.
  # Initialization finished.
  #
  #
  # +----------------------------------------------+
  # |               Training model 0               |
  # +----------------------------------------------+
  # Using TensorFlow backend.
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           0            |   4.180499947071075    |         0.646          |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |               Training model 1               |
  # +----------------------------------------------+
  # Epoch-1, Current Metric - 0:  21%|██████▍                       | 10/47 [00:00<00:01, 27.22 batch/s]
  # Current model size is too big. Discontinuing training this model to search for other models.
  #
  #
  # +----------------------------------------------+
  # |               Training model 2               |
  # +----------------------------------------------+
  # Epoch-1, Current Metric - 0:   0%|                                       | 0/47 [00:00<?, ? batch/s]
  # Current model size is too big. Discontinuing training this model to search for other models.
  #
  #
  # +----------------------------------------------+
  # |               Training model 3               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           3            |   4.236238491535187    |         0.668          |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |               Training model 4               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           4            |   4.188107478618622    |   0.6660000000000001   |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |               Training model 5               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           5            |   4.295066010951996    |         0.6632         |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |               Training model 6               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           6            |   4.163611340522766    |         0.6908         |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |               Training model 7               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           7            |   4.061753404140473    |   0.6748000000000001   |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |               Training model 8               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           8            |   4.495533287525177    |         0.6876         |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |               Training model 9               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           9            |   4.294030010700226    |   0.6824000000000001   |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |              Training model 10               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           10           |   4.310236525535584    |   0.7060000000000001   |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |              Training model 11               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           11           |   4.810216438770294    |         0.6724         |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |              Training model 12               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           12           |   4.930233514308929    |   0.6688000000000001   |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |              Training model 13               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           13           |   4.858993422985077    |   0.6816000000000001   |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |              Training model 14               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           14           |   4.861325883865357    |         0.6884         |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |              Training model 15               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           15           |   4.662096703052521    |         0.6896         |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |              Training model 16               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           16           |   4.792961382865906    |   0.6908000000000001   |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |              Training model 17               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           17           |   4.677005541324616    |         0.682          |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |              Training model 18               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           18           |   4.631161653995514    |         0.6916         |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |              Training model 19               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           19           |   4.666175007820129    |   0.6988000000000001   |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |              Training model 20               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           20           |   4.998566126823425    |         0.6984         |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |              Training model 21               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           21           |   4.308427512645721    |         0.7012         |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |              Training model 22               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           22           |   4.651701498031616    |         0.6452         |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |              Training model 23               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           23           |   4.820626270771027    |   0.6828000000000001   |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |              Training model 24               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           24           |   4.857317698001862    |   0.6812000000000001   |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |              Training model 25               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           25           |   4.555790698528289    |         0.6792         |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |              Training model 26               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           26           |   5.107322096824646    |         0.6668         |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |              Training model 27               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           27           |   4.783412170410156    |   0.7108000000000001   |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |              Training model 28               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           28           |    4.81434885263443    |   0.7011999999999999   |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |              Training model 29               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           29           |   4.861539125442505    |         0.7132         |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |              Training model 30               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           30           |   4.729412198066711    |         0.7344         |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |              Training model 31               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           31           |   5.048281860351563    |         0.718          |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |              Training model 32               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           32           |   5.429461920261383    |         0.7132         |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |              Training model 33               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           33           |   5.026830697059632    |         0.7232         |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |              Training model 34               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           34           |   5.176936650276184    |         0.702          |
  # +--------------------------------------------------------------------------+
  #
  #
  # +----------------------------------------------+
  # |              Training model 35               |
  # +----------------------------------------------+
  #
  # No loss decrease after 5 epochs.
  #
  #
  # Saving model.
  # +--------------------------------------------------------------------------+
  # |        Model ID        |          Loss          |      Metric Value      |
  # +--------------------------------------------------------------------------+
  # |           35           |   4.902780878543854    |   0.7163999999999999   |
  # +--------------------------------------------------------------------------+
}

# Evaluate before final fit
clf %>% evaluate(x_test, y_test)

# [1] 0.7312423

clf %>% export_autokeras_model("emoji_ds_bf.pkl")
# clf %>% export_keras_model("emoji_ds_bf.h5")

# Get the best trained model
clf %>%
  final_fit(
    x_train, y_train, x_test, y_test,
    retrain = TRUE, time_limit = 20 * 60
  )

## test model

# And use it to evaluate
clf %>% evaluate(x_test, y_test)

## [1] 0.7244772

# get the Keras model to work with the Keras R library
# get_keras_model(clf)

## save models

clf %>% export_autokeras_model("emoji_ds.pkl")
# clf %>% export_keras_model("emoji_ds.h5")