tests/testthat/test-s3-minio.R

In arrow: Integration to 'Apache' 'Arrow'

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

skip_if_not_available("s3")
skip_if_not(nzchar(Sys.which("minio")), message = "minio is not installed.")

library(dplyr)

minio_dir <- Sys.getenv("MINIO_DATA_DIR", tempfile())
minio_key <- "minioadmin"
minio_secret <- "minioadmin"
minio_port <- Sys.getenv("MINIO_PORT", "9000")

# Start minio server
dir.create(minio_dir, showWarnings = FALSE)
pid_minio <- sys::exec_background("minio", c("server", minio_dir, "--address", sprintf(":%s", minio_port)),
  std_out = FALSE
)
withr::defer(tools::pskill(pid_minio))

# Helper function for minio URIs
minio_uri <- function(...) {
  template <- "s3://%s:%s@%s?scheme=http&endpoint_override=localhost%s%s"
  sprintf(template, minio_key, minio_secret, minio_path(...), "%3A", minio_port)
}
minio_path <- function(...) paste(now, ..., sep = "/")

# Create a "bucket" on minio for this test run, which we'll delete when done.
fs <- S3FileSystem$create(
  access_key = minio_key,
  secret_key = minio_secret,
  scheme = "http",
  endpoint_override = paste0("localhost:", minio_port),
  allow_bucket_creation = TRUE,
  allow_bucket_deletion = TRUE
)
limited_fs <- S3FileSystem$create(
  access_key = minio_key,
  secret_key = minio_secret,
  scheme = "http",
  endpoint_override = paste0("localhost:", minio_port),
  allow_bucket_creation = FALSE,
  allow_bucket_deletion = FALSE
)
now <- as.character(as.numeric(Sys.time()))
fs$CreateDir(now)
# Clean up when we're all done
withr::defer(fs$DeleteDir(now))

test_filesystem("s3", fs, minio_path, minio_uri)

test_that("CreateDir fails on bucket if allow_bucket_creation=False", {
  now_tmp <- paste0(now, "-test-fail-delete")
  fs$CreateDir(now_tmp)

  expect_error(limited_fs$CreateDir("should-fail"))
  expect_error(limited_fs$DeleteDir(now_tmp))
})

test_that("S3FileSystem input validation", {
  expect_error(
    S3FileSystem$create(access_key = "foo"),
    "Key authentication requires both access_key and secret_key"
  )
  expect_error(
    S3FileSystem$create(secret_key = "foo"),
    "Key authentication requires both access_key and secret_key"
  )
  expect_error(
    S3FileSystem$create(session_token = "foo"),
    paste0(
      "In order to initialize a session with temporary credentials, ",
      "both secret_key and access_key must be provided ",
      "in addition to session_token."
    )
  )
  expect_error(
    S3FileSystem$create(access_key = "foo", secret_key = "asdf", anonymous = TRUE),
    'Cannot specify "access_key" and "secret_key" when anonymous = TRUE'
  )
  expect_error(
    S3FileSystem$create(access_key = "foo", secret_key = "asdf", role_arn = "qwer"),
    "Cannot provide both key authentication and role_arn"
  )
  expect_error(
    S3FileSystem$create(access_key = "foo", secret_key = "asdf", external_id = "qwer"),
    'Cannot specify "external_id" without providing a role_arn string'
  )
  expect_error(
    S3FileSystem$create(external_id = "foo"),
    'Cannot specify "external_id" without providing a role_arn string'
  )
})

test_that("Confirm s3_bucket works with endpoint_override", {
  bucket <- s3_bucket(
    now,
    access_key = minio_key,
    secret_key = minio_secret,
    scheme = "http",
    endpoint_override = paste0("localhost:", minio_port)
  )

  expect_r6_class(bucket, "SubTreeFileSystem")

  os <- bucket$OpenOutputStream("bucket-test.csv")
  write_csv_arrow(example_data, os)
  os$close()
  expect_true("bucket-test.csv" %in% bucket$ls())
  bucket$DeleteFile("bucket-test.csv")
})

# Cleanup
withr::deferred_run()