Nothing
#' Read a `.npy` file
#'
#' @param file Path to the `.npy` file
#' @param lazy If `TRUE`, and `file` is a path (not a connection), the data
#' payload is memory-mapped rather than read into memory upfront via
#' [readBin()]. This limits the number of copies in memory when the npy file
#' contains types native to R. Requires a POSIX platform; ignored (with a
#' warning) otherwise. Defaults to `FALSE`.
#' @param ... Ignored. Reserved for future use.
#'
#' @returns An array containing the data from the `.npy` file
#'
#' @export
#'
#' @examples
#' # Array of integers. NumPy "<i4" dtype
#' read_npy(
#' system.file("extdata", "test_int32.npy", package = "grumpy")
#' )
#'
#' # Array of logicals. NumPy "|b1" dtype
#' read_npy(
#' system.file("extdata", "test_bool.npy", package = "grumpy")
#' )
#'
#' # Array of numerics. NumPy "<f8" dtype
#' read_npy(
#' system.file("extdata", "test_float64.npy", package = "grumpy")
#' )
#'
#' # Array of strings. NumPy "<U" dtype
#' read_npy(
#' system.file("extdata", "test_str_unicode.npy", package = "grumpy")
#' )
#'
#' # "Array" of lists. Numpy structured dtype
#' read_npy(
#' system.file("extdata", "test_structured.npy", package = "grumpy")
#' )
read_npy <- function(file, lazy = FALSE, ...) {
chkDots(...)
if (is.character(file)) {
if (!file.exists(file)) {
stop("File does not exist: ", file, call. = FALSE)
}
con <- file(file, "rb")
on.exit(close(con))
} else if (inherits(file, "connection")) {
if (lazy) {
warning(
"`lazy = TRUE` is only supported when `file` is a path; ",
"ignoring it for connections.",
call. = FALSE
)
lazy <- FALSE
}
con <- file
} else {
stop(
"Invalid bytes: file must be a character string or a connection.",
call. = FALSE
)
}
# Read the header
magic_string <- readBin(con, "raw", n = 6L)
if (!identical(magic_string, charToRaw("\x93NUMPY"))) {
stop("Not a valid .npy file: ", file, call. = FALSE)
}
format_version <- readBin(
con,
"integer",
n = 2L,
size = 1L,
endian = "little"
)
header_len <- if (format_version[1L] == 1L) {
readBin(con, "integer", n = 1L, size = 2L, endian = "little")
} else if (format_version[1L] %in% c(2L, 3L)) {
readBin(con, "integer", n = 1L, size = 4L, endian = "little")
} else {
stop("Unsupported .npy version: ", format_version[1L], call. = FALSE)
}
header <- parse_npy_descr(readBin(con, "raw", n = header_len))
if (any(header$base_type == "py_object")) {
stop(
"This file contains a Python object array. ",
"Reading .npy files with Python object arrays is not supported. ",
"A common reason for this is when the file is saved with ",
"`np.save(..., allow_pickle=True)`. ",
call. = FALSE
)
}
# Read the data
num_elements <- prod(header$shape)
n_bytes <- sum(num_elements * header$nbytes)
if (lazy) {
# Offset of the payload is wherever the connection currently sits:
# 6 (magic) + 2 (version) + header_len_field_size (2 or 4) + header_len
payload_offset <- seek(con, origin = "current")
# When lazy = TRUE, we know `file` is a path, not a connection, so we can
# mmap it directly.
bytes <- mmap_raw(file, offset = payload_offset, length = n_bytes)
} else {
bytes <- readBin(con, "raw", n = n_bytes)
}
convert_bytes_to_array(
bytes,
what = header$base_type,
shape = header$shape,
size = header$nbytes,
endian = header$endian
)
}
#' Create an ALTREP raw vector backed by a memory-mapped file slice
#'
#' Internal helper used by [read_npy()] when `lazy = TRUE`. The returned
#' object behaves like an ordinary raw vector (as produced by [readBin()]),
#' but its bytes are only paged in from disk on first access, and no R-level
#' allocation happens for the full payload upfront. It can be passed to
#' [convert_bytes_to_array()] exactly like a materialized raw vector.
#'
#' @param path Path to the file to map.
#' @param offset Byte offset of the start of the desired slice.
#' @param length Number of bytes to expose.
#'
#' @returns An ALTREP raw vector of length `length`.
#'
#' @noRd
mmap_raw <- function(path, offset, length) {
.Call(
C_grumpy_make_mmap_raw,
path.expand(path),
as.double(offset),
as.double(length),
PACKAGE = "grumpy"
)
}
parse_npy_descr <- function(bytes) {
# TODO: If I understand correctly, fortranarray in python are still displayed
# the same way as regular arrays, but with a different order in memory.
# It is not related to the way the data is stored in the file, nor the way
# it appears to the user.
# We just ignore it, at least for now.
header <- bytes |>
rawToChar() |>
py_dict_to_r_list()
parsed_descr <- parse_npy_datatype(header$descr)
return(list(
endian = parsed_descr$endian,
base_type = parsed_descr$base_type,
nbytes = parsed_descr$nbytes,
fortran_order = header$fortran_order,
shape = header$shape
))
}
#' Parse a NumPy Array-protocol type string
#'
#' @param descr A NumPy dtype description string, or a list of such strings for
#' structured dtypes
#'
#' @returns A list containing the parsed data type information, including the
#' base type, the number of bytes, and the endianness
#'
#' @details
#' If a `list` is passed to `descr`, each element can be of length 1, or of
#' length 2 in which case the first element corresponds to the name of the field
#' and the second to its dtype.
#'
#' @export
#'
#' @examples
#' parse_npy_datatype(">i8")
#' parse_npy_datatype("|b1")
#' # A structured datatype where each element has 3 components, all integers,
#' # named "r", "g" and "b".
#' parse_npy_datatype(list(c("r", "<i8"), c("g", "<i8"), c("b", "<i8")))
#'
parse_npy_datatype <- function(descr) {
if (is.list(descr)) {
# structured data type
types <- lapply(descr, function(field) {
parse_npy_datatype(field[[2L]])
})
nms <- vapply(
descr,
function(field) {
field[[1L]]
},
character(1L)
)
return(
list(
base_type = setNames(
vapply(types, function(x) x$base_type, character(1L)),
nms
),
nbytes = vapply(types, function(x) x$nbytes, integer(1L)),
endian = vapply(types, function(x) x$endian, character(1L))
)
)
}
if (startsWith(descr, "|S")) {
return(
list(
base_type = "string",
nbytes = as.integer(substring(descr, 3L)),
endian = NA_character_
)
)
}
if (startsWith(descr, "<U") || startsWith(descr, ">U")) {
charlen <- as.integer(substring(descr, 3L))
return(
list(
base_type = "unicode",
nbytes = charlen * 4L,
endian = if (startsWith(descr, "<")) "little" else "big"
)
)
}
entry <- supported_types[[descr]]
if (is.null(entry)) {
stop("Unsupported data type: ", descr, call. = FALSE)
}
return(list(
base_type = entry$base_type,
nbytes = entry$nbytes,
endian = entry$endian
))
}
#' Convert raw bytes to an R array based on the specified data type information
#'
#' This is a replacement for [readBin()] that can handle the various data types
#' and endianness specified in the .npy file header.
#'
#' @param bytes A raw vector containing the bytes to convert
#' @param what A character specifying the base type to convert to (e.g.,
#' `"float"`, `"int"`, `"string"`, etc.)
#' @param shape A numeric vector with desired shape of the output array
#' @param size A numeric value with the number of bytes per element for the
#' specified type
#' @param endian The endianness of the data (`"little"`, `"big"`, or `NA` for
#' single-byte types)
#'
#' @returns An R array containing the converted data, with the specified shape
#' and data type.
#'
#' @export
#'
#' @examples
#' x <- matrix(c(3L, 6L, 2L, 1L, 12L, 0L), nrow = 2, ncol = 3)
#' x
#'
#' y <- writeBin(c(x), raw()) |>
#' convert_bytes_to_array(
#' "int",
#' shape = c(2L, 3L),
#' size = 4L,
#' endian = "little"
#' )
#' y
#' dim(y)
#' is.array(y)
#' storage.mode(y)
#'
convert_bytes_to_array <- function(bytes, what, shape, size, endian) {
if (length(what) > 1L) {
# structured datatype
record_size <- sum(size)
n_records <- prod(shape)
# Byte offset of each field within one record
field_start <- c(0L, cumsum(size)[-length(size)])
# Convert each field via strided index extraction
res_fields <- vector("list", length(what)) |> setNames(names(what))
for (i in seq_along(what)) {
starts <- seq(
field_start[[i]] + 1L,
by = record_size,
length.out = n_records
)
idx <- rep(starts, each = size[[i]]) + seq_len(size[[i]]) - 1L
res_fields[[i]] <- convert_bytes_to_array(
bytes[idx],
what = what[[i]],
shape = NULL,
size = size[[i]],
endian = endian[[i]]
)
}
# Final list-transpose
res <- vector("list", n_records)
for (j in seq_len(n_records)) {
res[[j]] <- lapply(res_fields, `[[`, j)
}
dim(res) <- shape
return(res)
}
if (is.na(endian)) {
endian <- .Platform$endian
}
# FIXME: optimize this
if (what != "unicode" && endian != .Platform$endian) {
ind <- rep_len(rev(seq_len(size)), length(bytes)) +
(seq_along(bytes) - 1L) %/% size * size
bytes <- bytes[ind]
}
res <- .Call(
C_type_convert,
bytes,
what,
size,
shape,
endian,
PACKAGE = "grumpy"
)
return(res)
}
py_dict_to_r_list <- function(x) {
x |>
# Logicals
gsub("\\bNone\\b", "NA", x = _) |>
gsub("\\bTrue\\b", "TRUE", x = _) |>
gsub("\\bFalse\\b", "FALSE", x = _) |>
# Trailing commas (before closing brackets/braces)
gsub(",\\s*([]})])", "\\1", x = _) |>
# Dict (single-quoted) keys to list names.
# Keys are composed of:
# - anything that is not the quote itself or a backslash,
# - a backslash followed by any character (to allow for escaped quotes).
gsub("'(([^'\\\\]|\\\\.)*)'\\s*:", "'\\1' =", x = _) |>
# Dict (double-quoted) keys to list names.
gsub('"(([^"\\\\]|\\\\.)*)"\\s*:', '"\\1" =', x = _) |>
# Tuples to c(...)
gsub("(", "c(", x = _, fixed = TRUE) |>
# Lists to list(...)
gsub("[", "list(", x = _, fixed = TRUE) |>
gsub("{", "list(", x = _, fixed = TRUE) |>
# Closing brackets/braces
gsub("]", ")", x = _, fixed = TRUE) |>
gsub("}", ")", x = _, fixed = TRUE) |>
# Evaluate the resulting R expression
parse(text = _) |>
eval()
}
Any scripts or data that you put into this service are public.
Add the following code to your website.
For more information on customizing the embed code, read Embedding Snippets.