tests/testthat/test_LIME.R

In bips-hb/innsight: Get the Insights of Your Neural Network

test_that("LIME: General errors", {
  library(neuralnet)

  # Fit model
  model <- neuralnet(Species ~ Petal.Length + Petal.Width, iris,
                     linear.output = FALSE)
  data <- iris[, c(3,4)]

  expect_error(LIME$new()) # missing converter
  expect_error(LIME$new(model)) # missing data
  expect_error(LIME$new(NULL, data[1:2, ], data )) # no output_type
  expect_error(LIME$new(NULL, data[1:2, ], data, output_type = "regression")) # no pred_fun
  expect_error(LIME$new(NULL, data[1:2, ], data,
                        output_type = "regression",
                        perd_fun = function(newdata, ...) newdata))

  LIME$new(model, data[1:2, ], data) # successful run
  expect_error(LIME$new(model, data[1:2, ], data, output_type = "ds")) # wrong output_type
  expect_error(LIME$new(model, data[1:2, ], data, pred_fun = identity)) # wrong pred_fun
  expect_error(LIME$new(model, data[1:2, ], data, output_idx = c(1,4))) # wrong output_idx
  LIME$new(model, data[1:2, ], data, output_idx = c(2))
  expect_error(LIME$new(model, data[1:2, ], data, input_dim = c(1))) # wrong input_dim
  expect_error(LIME$new(model, data[1:2, ], data, input_names = c("a", "b", "d"))) # wrong input_names
  LIME$new(model, data[1:2, ], data, input_names = factor(c("a", "b")))
  expect_error(LIME$new(model, data[1:2, ], data, output_names = c("a", "d"))) # wrong output_names
  LIME$new(model, data[1:2, ], data, output_names = factor(c("a", "d", "c")))

  expect_error(LIME$new(model, data[1:2, ], data, output_idx = c(1, 10)))
  LIME$new(model, data[1:2, ], data, output_idx = c(1, 2))
  expect_error(LIME$new(model, data[1:2, ], data, output_idx = list(c(1, 10))))
  LIME$new(model, data[1:2, ], data, output_idx = list(c(1, 3)))
  expect_error(LIME$new(model, data[1:2, ], data, output_idx = list(NULL, c(1, 2))))
  expect_error(LIME$new(model, data[1:2, ], data, output_label = c(1, 2)))
  expect_error(LIME$new(model, data[1:2, ], data, output_label = c("A", "b")))
  LIME$new(model, data[1:2, ], data, output_label = c("setosa", "virginica"))
  LIME$new(model, data[1:2, ], data, output_label = as.factor(c("setosa", "virginica")))
  LIME$new(model, data[1:2, ], data, output_label = list(c("setosa", "virginica")))
  expect_error(LIME$new(model, data[1:2, ], data,
                        output_label =  c("setosa", "virginica"),
                        output_idx = c(1, 2)))
  LIME$new(model, data[1:2, ], data,
           output_label =  c("setosa", "virginica"),
           output_idx = c(1, 3))

  # Forwarding arguments to lime::explain
  lime <- LIME$new(model, data[1:10, ], data, n_permutations = 100, gower_power = 3)

  # get_result()
  res <- get_result(lime)
  expect_array(res)
  res <- get_result(lime, "data.frame")
  expect_data_frame(res)
  res <- get_result(lime, "torch_tensor")
  expect_class(res, "torch_tensor")

  # Plots

  # Non-existing data points
  expect_error(plot(lime, data_idx = c(1,11)))
  expect_error(boxplot(lime, data_idx = 1:11))
  # Non-existing class
  expect_error(plot(lime, output_idx = c(5)))
  expect_error(boxplot(lime, output_idx = c(5)))

  p <- plot(lime)
  boxp <- boxplot(lime)
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")
  p <- plot(lime, data_idx = 1:3)
  boxp <- boxplot(lime, data_idx = 1:4)
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")
  p <- plot(lime, data_idx = 1:3, output_idx = 1:3)
  boxp <- boxplot(lime, data_idx = 1:3, output_idx = 1:3)
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")
  boxp <- boxplot(lime, ref_data_idx = c(4))

  # plotly
  library(plotly)

  p <- plot(lime, as_plotly = TRUE)
  boxp <- boxplot(lime, as_plotly = TRUE)
  expect_s4_class(p, "innsight_plotly")
  expect_s4_class(boxp, "innsight_plotly")
  p <- plot(lime, data_idx = 1:3, as_plotly = TRUE)
  boxp <- boxplot(lime, data_idx = 1:4, as_plotly = TRUE, individual_max = 2,
                  individual_data_idx = c(1,2,5,6))
  expect_s4_class(p, "innsight_plotly")
  expect_s4_class(boxp, "innsight_plotly")
  p <- plot(lime, data_idx = 1:3, output_idx = 1:3, as_plotly = TRUE)
  boxp <- boxplot(lime, data_idx = 1:5, output_idx = 1:3, as_plotly = TRUE)
  expect_s4_class(p, "innsight_plotly")
  expect_s4_class(boxp, "innsight_plotly")


})

test_that("LIME: Dense-Net (Neuralnet)", {
  library(neuralnet)
  library(torch)

  data(iris)
  data <- iris[sample.int(150, size = 10), -5]
  nn <- neuralnet(Species ~ .,
                  iris,
                  linear.output = FALSE,
                  hidden = c(10, 8), act.fct = "tanh", rep = 1, threshold = 0.5
  )

  # Normal model
  lime <- LIME$new(nn, data[1:2, ], data)
  expect_equal(dim(lime$get_result()), c(2, 4, 3))
  p <- plot(lime, output_idx = c(2,3))
  boxp <- boxplot(lime, output_idx = c(2,3))
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")

  # Converter
  conv <- Converter$new(nn)
  lime <- LIME$new(conv, data[1:2, ], data)
  expect_equal(dim(lime$get_result()), c(2, 4, 3))
  p <- plot(lime, output_idx = c(2,3))
  boxp <- boxplot(lime, output_idx = c(2,3))
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")
})


test_that("LIME: Dense-Net (keras)", {
  library(keras)
  library(torch)

  # Classification -------------------------------------------------------------
  data <- matrix(rnorm(4 * 10), nrow = 10)
  model <- keras_model_sequential()
  model %>%
    layer_dense(units = 16, activation = "relu", input_shape = c(4)) %>%
    layer_dense(units = 8, activation = "tanh") %>%
    layer_dense(units = 3, activation = "softmax")

  # Normal model
  lime <- LIME$new(model, data[1:2, ], data)
  expect_equal(dim(lime$get_result()), c(2, 4, 3))
  p <- plot(lime, output_idx = c(1,3))
  boxp <- boxplot(lime, output_idx = c(1,3))
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")

  # Converter
  conv <- Converter$new(model)
  lime <- LIME$new(conv, data[1:2, ], data)
  expect_equal(dim(lime$get_result()), c(2, 4, 3))
  p <- plot(lime, output_idx = c(1,3))
  boxp <- boxplot(lime, output_idx = c(1,3))
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")

  # Regression -----------------------------------------------------------------
  data <- matrix(rnorm(4 * 10), nrow = 10)
  model <- keras_model_sequential()
  model %>%
    layer_dense(units = 16, activation = "relu", input_shape = c(4)) %>%
    layer_dense(units = 8, activation = "tanh") %>%
    layer_dense(units = 2, activation = "linear")

  # Normal model
  lime <- LIME$new(model, data[1:2, ], data)
  expect_equal(dim(lime$get_result()), c(2, 4, 2))
  p <- plot(lime, output_idx = c(2))
  boxp <- boxplot(lime, output_idx = c(2))
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")

  # Converter
  conv <- Converter$new(model)
  lime <- LIME$new(conv, data[1:2, ], data)
  expect_equal(dim(lime$get_result()), c(2, 4, 2))
  p <- plot(lime, output_idx = c(2))
  boxp <- boxplot(lime, output_idx = c(2))
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")

  # Test get_result -------------------------------------------------------------
  res_array <- lime$get_result()
  expect_true(is.array(res_array))
  res_dataframe <- lime$get_result(type = "data.frame")
  expect_true(is.data.frame(res_dataframe))
  res_torch <- lime$get_result(type = "torch.tensor")
  expect_true(inherits(res_torch, "torch_tensor"))
  expect_error(lime$get_result(type = "adsf"))
})


test_that("LIME: Conv1D-Net (keras)", {
  library(keras)
  library(torch)

  # Classification -------------------------------------------------------------
  data <- array(rnorm(4 * 64 * 3), dim = c(4, 64, 3))
  model <- keras_model_sequential()
  model %>%
    layer_conv_1d(
      input_shape = c(64, 3), kernel_size = 16, filters = 8,
      activation = "softplus"
    ) %>%
    layer_conv_1d(kernel_size = 16, filters = 4, activation = "tanh") %>%
    layer_conv_1d(kernel_size = 16, filters = 2, activation = "relu") %>%
    layer_flatten() %>%
    layer_dense(units = 64, activation = "relu") %>%
    layer_dense(units = 16, activation = "relu") %>%
    layer_dense(units = 1, activation = "sigmoid")

  # Normal model
  lime <- LIME$new(model, data[1:2,, ], data, channels_first = FALSE)
  expect_equal(dim(lime$get_result()), c(2, 64, 3, 1))
  p <- plot(lime)
  boxp <- boxplot(lime)
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")

  # Converter
  conv <- Converter$new(model)
  lime <- LIME$new(conv, data[1:2,, ], data, channels_first = FALSE)
  expect_equal(dim(lime$get_result()), c(2, 64, 3, 1))
  p <- plot(lime)
  boxp <- boxplot(lime)
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")

  # Regression -----------------------------------------------------------------
  data <- array(rnorm(4 * 64 * 3), dim = c(4, 64, 3))
  model <- keras_model_sequential()
  model %>%
    layer_conv_1d(
      input_shape = c(64, 3), kernel_size = 16, filters = 8,
      activation = "softplus"
    ) %>%
    layer_conv_1d(kernel_size = 16, filters = 4, activation = "tanh") %>%
    layer_conv_1d(kernel_size = 16, filters = 2, activation = "relu") %>%
    layer_flatten() %>%
    layer_dense(units = 64, activation = "relu") %>%
    layer_dense(units = 16, activation = "relu") %>%
    layer_dense(units = 2, activation = "linear")

  # Normal model
  lime <- LIME$new(model, data[1:2,, ], data, channels_first = FALSE)
  expect_equal(dim(lime$get_result()), c(2, 64, 3, 2))
  p <- plot(lime, output_idx = c(2))
  boxp <- boxplot(lime, output_idx = c(2))
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")

  # Converter
  conv <- Converter$new(model)
  lime <- LIME$new(conv, data[1:2,, ], data, channels_first = FALSE)
  expect_equal(dim(lime$get_result()), c(2, 64, 3, 2))
  p <- plot(lime, output_idx = c(2))
  boxp <- boxplot(lime, output_idx = c(2))
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")

  # Test get_result ------------------------------------------------------------
  res_array <- lime$get_result()
  expect_true(is.array(res_array))
  res_dataframe <- lime$get_result(type = "data.frame")
  expect_true(is.data.frame(res_dataframe))
  res_torch <- lime$get_result(type = "torch.tensor")
  expect_true(inherits(res_torch, "torch_tensor"))
})

test_that("LIME: Conv2D-Net (keras)", {
  library(keras)
  library(torch)

  # Classification -------------------------------------------------------------
  data <- array(rnorm(4 * 10 * 10 * 3), dim = c(4, 10, 10, 3))
  model <- keras_model_sequential()
  model %>%
    layer_conv_2d(
      input_shape = c(10, 10, 3), kernel_size = 4, filters = 8,
      activation = "softplus"
    ) %>%
    layer_conv_2d(kernel_size = 4, filters = 2, activation = "relu") %>%
    layer_flatten() %>%
    layer_dense(units = 16, activation = "relu") %>%
    layer_dense(units = 1, activation = "sigmoid")

  # Normal model
  lime <- LIME$new(model, data[1:2,,, ], data, channels_first = FALSE)
  expect_equal(dim(lime$get_result()), c(2, 10, 10, 3, 1))
  p <- plot(lime)
  boxp <- plot_global(lime)
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")

  # Converter
  conv <- Converter$new(model)
  lime <- LIME$new(conv, data[1:2,,, ], data, channels_first = FALSE)
  expect_equal(dim(lime$get_result()), c(2, 10, 10, 3, 1))
  p <- plot(lime)
  boxp <- plot_global(lime)
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")

  # Regression -----------------------------------------------------------------
  data <- array(rnorm(4 * 10 * 10 * 3), dim = c(4, 10, 10, 3))
  model <- keras_model_sequential()
  model %>%
    layer_conv_2d(
      input_shape = c(10, 10, 3), kernel_size = 4, filters = 8,
      activation = "softplus"
    ) %>%
    layer_conv_2d(kernel_size = 4, filters = 2, activation = "relu") %>%
    layer_flatten() %>%
    layer_dense(units = 16, activation = "relu") %>%
    layer_dense(units = 2, activation = "linear")

  # Normal model
  lime <- LIME$new(model, data[1:2,,, ], data, channels_first = FALSE)
  expect_equal(dim(lime$get_result()), c(2, 10, 10, 3, 2))
  p <- plot(lime, output_idx = c(2))
  boxp <- plot_global(lime, output_idx = c(2))
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")

  # Converter
  conv <- Converter$new(model)
  lime <- LIME$new(conv, data[1:2,,, ], data, channels_first = FALSE)
  expect_equal(dim(lime$get_result()), c(2, 10, 10, 3, 2))
  p <- plot(lime, output_idx = c(2))
  boxp <- plot_global(lime, output_idx = c(2))
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")

  # Test get_result ------------------------------------------------------------
  res_array <- lime$get_result()
  expect_true(is.array(res_array))
  res_dataframe <- lime$get_result(type = "data.frame")
  expect_true(is.data.frame(res_dataframe))
  res_torch <- lime$get_result(type = "torch.tensor")
  expect_true(inherits(res_torch, "torch_tensor"))
})


test_that("LIME: Dense-Net (torch)", {
  library(torch)

  # Classification -------------------------------------------------------------
  data <- matrix(rnorm(4 * 10), nrow = 10)
  model <- nn_sequential(
    nn_linear(4, 16),
    nn_relu(),
    nn_linear(16, 3),
    nn_softmax(dim = -1)
  )

  # Normal model
  lime <- LIME$new(model, data[1:2, ], data)
  expect_equal(dim(lime$get_result()), c(2, 4, 3))
  p <- plot(lime, output_idx = c(1,3))
  boxp <- boxplot(lime, output_idx = c(1,3))
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")

  # Converter
  conv <- Converter$new(model, input_dim = c(4))
  lime <- LIME$new(conv, data[1:2, ], data)
  expect_equal(dim(lime$get_result()), c(2, 4, 3))
  p <- plot(lime, output_idx = c(1,3))
  boxp <- boxplot(lime, output_idx = c(1,3))
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")

  # Regression -----------------------------------------------------------------
  data <- matrix(rnorm(4 * 10), nrow = 10)
  model <- nn_sequential(
    nn_linear(4, 16),
    nn_relu(),
    nn_linear(16, 2)
  )

  # Normal model
  lime <- LIME$new(model, data[1:2, ], data)
  expect_equal(dim(lime$get_result()), c(2, 4, 2))
  p <- plot(lime, output_idx = c(2))
  boxp <- boxplot(lime, output_idx = c(2))
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")

  # Converter
  conv <- Converter$new(model, input_dim = c(4))
  lime <- LIME$new(conv, data[1:2, ], data)
  expect_equal(dim(lime$get_result()), c(2, 4, 2))
  p <- plot(lime, output_idx = c(2))
  boxp <- boxplot(lime, output_idx = c(2))
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")

  # Test get_result -------------------------------------------------------------
  res_array <- lime$get_result()
  expect_true(is.array(res_array))
  res_dataframe <- lime$get_result(type = "data.frame")
  expect_true(is.data.frame(res_dataframe))
  res_torch <- lime$get_result(type = "torch.tensor")
  expect_true(inherits(res_torch, "torch_tensor"))
})


test_that("LIME: Conv1D-Net (torch)", {
  library(torch)

  # Classification -------------------------------------------------------------
  data <- array(rnorm(4 * 64 * 3), dim = c(4, 3, 64))
  model <- nn_sequential(
    nn_conv1d(3, 8, 16),
    nn_softplus(),
    nn_conv1d(8, 4, 16),
    nn_tanh(),
    nn_conv1d(4, 2, 16),
    nn_relu(),
    nn_flatten(),
    nn_linear(38, 16),
    nn_relu(),
    nn_linear(16, 1),
    nn_sigmoid()
  )

  # Normal model
  lime <- LIME$new(model, data[1:2,, ], data, channels_first = TRUE)
  expect_equal(dim(lime$get_result()), c(2, 3, 64, 1))
  p <- plot(lime)
  boxp <- boxplot(lime)
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")

  # Converter
  conv <- Converter$new(model, input_dim = c(3, 64))
  lime <- LIME$new(conv, data[1:2,, ], data, channels_first = TRUE)
  expect_equal(dim(lime$get_result()), c(2, 3, 64, 1))
  p <- plot(lime)
  boxp <- boxplot(lime)
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")

  # Regression -----------------------------------------------------------------
  data <- array(rnorm(4 * 64 * 3), dim = c(4, 3, 64))
  model <- nn_sequential(
    nn_conv1d(3, 8, 16),
    nn_softplus(),
    nn_conv1d(8, 4, 16),
    nn_tanh(),
    nn_conv1d(4, 2, 16),
    nn_relu(),
    nn_flatten(),
    nn_linear(38, 16),
    nn_relu(),
    nn_linear(16, 2)
  )

  # Normal model
  lime <- LIME$new(model, data[1:2,, ], data, channels_first = TRUE)
  expect_equal(dim(lime$get_result()), c(2, 3, 64, 2))
  p <- plot(lime, output_idx = c(2))
  boxp <- boxplot(lime, output_idx = c(2))
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")

  # Converter
  conv <- Converter$new(model, input_dim = c(3, 64))
  lime <- LIME$new(conv, data[1:2,, ], data, channels_first = TRUE)
  expect_equal(dim(lime$get_result()), c(2, 3, 64, 2))
  p <- plot(lime, output_idx = c(2))
  boxp <- boxplot(lime, output_idx = c(2))
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")

  # Test get_result ------------------------------------------------------------
  res_array <- lime$get_result()
  expect_true(is.array(res_array))
  res_dataframe <- lime$get_result(type = "data.frame")
  expect_true(is.data.frame(res_dataframe))
  res_torch <- lime$get_result(type = "torch.tensor")
  expect_true(inherits(res_torch, "torch_tensor"))
})

test_that("LIME: Conv2D-Net (torch)", {
  library(keras)
  library(torch)

  # Classification -------------------------------------------------------------
  data <- array(rnorm(4 * 10 * 10 * 3), dim = c(4, 3, 10, 10))
  model <- nn_sequential(
    nn_conv2d(3, 8, c(4, 4)),
    nn_softplus(),
    nn_conv2d(8, 2, c(4, 4)),
    nn_relu(),
    nn_flatten(),
    nn_linear(32, 16),
    nn_relu(),
    nn_linear(16, 1),
    nn_sigmoid()
  )

  # Normal model
  lime <- LIME$new(model, data[1:2,,, ], data, channels_first = TRUE)
  expect_equal(dim(lime$get_result()), c(2, 3, 10, 10, 1))
  p <- plot(lime)
  boxp <- plot_global(lime)
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")

  # Converter
  conv <- Converter$new(model, input_dim = c(3, 10, 10))
  lime <- LIME$new(conv, data[1:2,,, ], data, channels_first = TRUE)
  expect_equal(dim(lime$get_result()), c(2, 3, 10, 10, 1))
  p <- plot(lime)
  boxp <- plot_global(lime)
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")

  # Regression -----------------------------------------------------------------
  data <- array(rnorm(4 * 10 * 10 * 3), dim = c(4, 3, 10, 10))
  model <- nn_sequential(
    nn_conv2d(3, 8, c(4, 4)),
    nn_softplus(),
    nn_conv2d(8, 2, c(4, 4)),
    nn_relu(),
    nn_flatten(),
    nn_linear(32, 16),
    nn_relu(),
    nn_linear(16, 2)
  )

  # Normal model
  lime <- LIME$new(model, data[1:2,,, ], data, channels_first = TRUE)
  expect_equal(dim(lime$get_result()), c(2, 3, 10, 10, 2))
  p <- plot(lime, output_idx = c(2))
  boxp <- plot_global(lime, output_idx = c(2))
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")

  # Converter
  conv <- Converter$new(model, input_dim = c(3, 10, 10))
  lime <- LIME$new(conv, data[1:2,,, ], data, channels_first = TRUE)
  expect_equal(dim(lime$get_result()), c(2, 3, 10, 10, 2))
  p <- plot(lime, output_idx = c(2))
  boxp <- plot_global(lime, output_idx = c(2))
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")

  # Test get_result ------------------------------------------------------------
  res_array <- lime$get_result()
  expect_true(is.array(res_array))
  res_dataframe <- lime$get_result(type = "data.frame")
  expect_true(is.data.frame(res_dataframe))
  res_torch <- lime$get_result(type = "torch.tensor")
  expect_true(inherits(res_torch, "torch_tensor"))
})


test_that("LIME: Keras multiple input or output layers", {
  library(keras)

  # Multiple input layers
  main_input <- layer_input(shape = c(10,10,2), name = 'main_input')
  lstm_out <- main_input %>%
    layer_conv_2d(2, c(2,2), activation = "relu") %>%
    layer_flatten() %>%
    layer_dense(units = 4)
  auxiliary_input <- layer_input(shape = c(5), name = 'aux_input')
  main_output <- layer_concatenate(c(lstm_out, auxiliary_input)) %>%
    layer_dense(units = 5, activation = 'tanh') %>%
    layer_dense(units = 3, activation = 'softmax', name = 'main_output')
  model <- keras_model(
    inputs = c(auxiliary_input, main_input),
    outputs = c(main_output)
  )
  data <- lapply(list(c(5), c(10,10,2)),
                 function(x) array(rnorm(10 * prod(x)), dim = c(10, x)))

  expect_error(LIME$new(model, data, data_ref = NULL))

  # Multiple output layers
  main_input <- layer_input(shape = c(10,10,2), name = 'main_input')
  lstm_out <- main_input %>%
    layer_conv_2d(2, c(2,2), activation = "relu") %>%
    layer_flatten() %>%
    layer_dense(units = 4)
  auxiliary_output <- lstm_out %>%
    layer_dense(units = 2, activation = 'softmax', name = 'aux_output')
  main_output <- lstm_out %>%
    layer_dense(units = 5, activation = 'tanh') %>%
    layer_dense(units = 3, activation = 'softmax', name = 'main_output')
  model <- keras_model(
    inputs = c(main_input),
    outputs = c(auxiliary_output, main_output)
  )
  data <- lapply(list(c(10,10,2)),
                 function(x) array(rnorm(10 * prod(x)), dim = c(10, x)))


  expect_error(LIME$new(model, data))
})

test_that("Custom model", {

  # Ranger model and iris dataset
  library(ranger)

  model <- ranger(Species ~ ., data = iris, probability = TRUE)

  pred_fun <- function(newdata, ...) {
    predict(model, newdata, ...)$predictions
  }

  lime <- LIME$new(model, iris[c(1,70, 111), -5], iris[, -5],
                   output_type = "classification",
                   pred_fun = pred_fun,
                   output_names = levels(iris$Species))

  res <- get_result(lime)
  expect_equal(dim(res), c(3, 4, 3))

  p <- plot(lime, output_idx = c(1, 3))
  boxp <- boxplot(lime, output_idx = c(1, 3))
  expect_s4_class(p, "innsight_ggplot2")
  expect_s4_class(boxp, "innsight_ggplot2")
})