R/read_npy.R

Defines functions py_dict_to_r_list convert_bytes_to_array parse_npy_datatype parse_npy_descr mmap_raw read_npy

Documented in convert_bytes_to_array parse_npy_datatype read_npy

#' Read a `.npy` file
#'
#' @param file Path to the `.npy` file
#' @param lazy If `TRUE`, and `file` is a path (not a connection), the data
#'   payload is memory-mapped rather than read into memory upfront via
#'   [readBin()]. This limits the number of copies in memory when the npy file
#'   contains types native to R. Requires a POSIX platform; ignored (with a
#'   warning) otherwise. Defaults to `FALSE`.
#' @param ... Ignored. Reserved for future use.
#'
#' @returns An array containing the data from the `.npy` file
#'
#' @export
#'
#' @examples
#' # Array of integers. NumPy "<i4" dtype
#' read_npy(
#'   system.file("extdata", "test_int32.npy", package = "grumpy")
#' )
#'
#' # Array of logicals. NumPy "|b1" dtype
#' read_npy(
#'   system.file("extdata", "test_bool.npy", package = "grumpy")
#' )
#'
#' # Array of numerics. NumPy "<f8" dtype
#' read_npy(
#'   system.file("extdata", "test_float64.npy", package = "grumpy")
#' )
#'
#' # Array of strings. NumPy "<U" dtype
#' read_npy(
#'   system.file("extdata", "test_str_unicode.npy", package = "grumpy")
#' )
#'
#' # "Array" of lists. Numpy structured dtype
#' read_npy(
#'   system.file("extdata", "test_structured.npy", package = "grumpy")
#' )
read_npy <- function(file, lazy = FALSE, ...) {
  chkDots(...)

  if (is.character(file)) {
    if (!file.exists(file)) {
      stop("File does not exist: ", file, call. = FALSE)
    }
    con <- file(file, "rb")
    on.exit(close(con))
  } else if (inherits(file, "connection")) {
    if (lazy) {
      warning(
        "`lazy = TRUE` is only supported when `file` is a path; ",
        "ignoring it for connections.",
        call. = FALSE
      )
      lazy <- FALSE
    }
    con <- file
  } else {
    stop(
      "Invalid bytes: file must be a character string or a connection.",
      call. = FALSE
    )
  }

  # Read the header
  magic_string <- readBin(con, "raw", n = 6L)
  if (!identical(magic_string, charToRaw("\x93NUMPY"))) {
    stop("Not a valid .npy file: ", file, call. = FALSE)
  }

  format_version <- readBin(
    con,
    "integer",
    n = 2L,
    size = 1L,
    endian = "little"
  )

  header_len <- if (format_version[1L] == 1L) {
    readBin(con, "integer", n = 1L, size = 2L, endian = "little")
  } else if (format_version[1L] %in% c(2L, 3L)) {
    readBin(con, "integer", n = 1L, size = 4L, endian = "little")
  } else {
    stop("Unsupported .npy version: ", format_version[1L], call. = FALSE)
  }

  header <- parse_npy_descr(readBin(con, "raw", n = header_len))

  if (any(header$base_type == "py_object")) {
    stop(
      "This file contains a Python object array. ",
      "Reading .npy files with Python object arrays is not supported. ",
      "A common reason for this is when the file is saved with ",
      "`np.save(..., allow_pickle=True)`. ",
      call. = FALSE
    )
  }

  # Read the data
  num_elements <- prod(header$shape)
  n_bytes <- sum(num_elements * header$nbytes)

  if (lazy) {
    # Offset of the payload is wherever the connection currently sits:
    # 6 (magic) + 2 (version) + header_len_field_size (2 or 4) + header_len
    payload_offset <- seek(con, origin = "current")
    # When lazy = TRUE, we know `file` is a path, not a connection, so we can
    # mmap it directly.
    bytes <- mmap_raw(file, offset = payload_offset, length = n_bytes)
  } else {
    bytes <- readBin(con, "raw", n = n_bytes)
  }

  convert_bytes_to_array(
    bytes,
    what = header$base_type,
    shape = header$shape,
    size = header$nbytes,
    endian = header$endian
  )
}

#' Create an ALTREP raw vector backed by a memory-mapped file slice
#'
#' Internal helper used by [read_npy()] when `lazy = TRUE`. The returned
#' object behaves like an ordinary raw vector (as produced by [readBin()]),
#' but its bytes are only paged in from disk on first access, and no R-level
#' allocation happens for the full payload upfront. It can be passed to
#' [convert_bytes_to_array()] exactly like a materialized raw vector.
#'
#' @param path Path to the file to map.
#' @param offset Byte offset of the start of the desired slice.
#' @param length Number of bytes to expose.
#'
#' @returns An ALTREP raw vector of length `length`.
#'
#' @noRd
mmap_raw <- function(path, offset, length) {
  .Call(
    C_grumpy_make_mmap_raw,
    path.expand(path),
    as.double(offset),
    as.double(length),
    PACKAGE = "grumpy"
  )
}

parse_npy_descr <- function(bytes) {
  # TODO: If I understand correctly, fortranarray in python are still displayed
  # the same way as regular arrays, but with a different order in memory.
  # It is not related to the way the data is stored in the file, nor the way
  # it appears to the user.
  # We just ignore it, at least for now.
  header <- bytes |>
    rawToChar() |>
    py_dict_to_r_list()

  parsed_descr <- parse_npy_datatype(header$descr)

  return(list(
    endian = parsed_descr$endian,
    base_type = parsed_descr$base_type,
    nbytes = parsed_descr$nbytes,
    fortran_order = header$fortran_order,
    shape = header$shape
  ))
}

#' Parse a NumPy Array-protocol type string
#'
#' @param descr A NumPy dtype description string, or a list of such strings for
#'   structured dtypes
#'
#' @returns A list containing the parsed data type information, including the
#'   base type, the number of bytes, and the endianness
#'
#' @details
#' If a `list` is passed to `descr`, each element can be of length 1, or of
#' length 2 in which case the first element corresponds to the name of the field
#' and the second to its dtype.
#'
#' @export
#'
#' @examples
#' parse_npy_datatype(">i8")
#' parse_npy_datatype("|b1")
#' # A structured datatype where each element has 3 components, all integers,
#' # named "r", "g" and "b".
#' parse_npy_datatype(list(c("r", "<i8"), c("g", "<i8"), c("b", "<i8")))
#'
parse_npy_datatype <- function(descr) {
  if (is.list(descr)) {
    # structured data type
    types <- lapply(descr, function(field) {
      parse_npy_datatype(field[[2L]])
    })
    nms <- vapply(
      descr,
      function(field) {
        field[[1L]]
      },
      character(1L)
    )
    return(
      list(
        base_type = setNames(
          vapply(types, function(x) x$base_type, character(1L)),
          nms
        ),
        nbytes = vapply(types, function(x) x$nbytes, integer(1L)),
        endian = vapply(types, function(x) x$endian, character(1L))
      )
    )
  }
  if (startsWith(descr, "|S")) {
    return(
      list(
        base_type = "string",
        nbytes = as.integer(substring(descr, 3L)),
        endian = NA_character_
      )
    )
  }
  if (startsWith(descr, "<U") || startsWith(descr, ">U")) {
    charlen <- as.integer(substring(descr, 3L))
    return(
      list(
        base_type = "unicode",
        nbytes = charlen * 4L,
        endian = if (startsWith(descr, "<")) "little" else "big"
      )
    )
  }

  entry <- supported_types[[descr]]
  if (is.null(entry)) {
    stop("Unsupported data type: ", descr, call. = FALSE)
  }

  return(list(
    base_type = entry$base_type,
    nbytes = entry$nbytes,
    endian = entry$endian
  ))
}

#' Convert raw bytes to an R array based on the specified data type information
#'
#' This is a replacement for [readBin()] that can handle the various data types
#' and endianness specified in the .npy file header.
#'
#' @param bytes A raw vector containing the bytes to convert
#' @param what A character specifying the base type to convert to (e.g.,
#'   `"float"`, `"int"`, `"string"`, etc.)
#' @param shape A numeric vector with desired shape of the output array
#' @param size A numeric value with the number of bytes per element for the
#'   specified type
#' @param endian The endianness of the data (`"little"`, `"big"`, or `NA` for
#'   single-byte types)
#'
#' @returns An R array containing the converted data, with the specified shape
#'    and data type.
#'
#' @export
#'
#' @examples
#' x <- matrix(c(3L, 6L, 2L, 1L, 12L, 0L), nrow = 2, ncol = 3)
#' x
#'
#' y <- writeBin(c(x), raw()) |>
#'   convert_bytes_to_array(
#'      "int",
#'      shape = c(2L, 3L),
#'      size = 4L,
#'      endian = "little"
#'   )
#' y
#' dim(y)
#' is.array(y)
#' storage.mode(y)
#'
convert_bytes_to_array <- function(bytes, what, shape, size, endian) {
  if (length(what) > 1L) {
    # structured datatype
    record_size <- sum(size)
    n_records <- prod(shape)

    # Byte offset of each field within one record
    field_start <- c(0L, cumsum(size)[-length(size)])

    # Convert each field via strided index extraction
    res_fields <- vector("list", length(what)) |> setNames(names(what))
    for (i in seq_along(what)) {
      starts <- seq(
        field_start[[i]] + 1L,
        by = record_size,
        length.out = n_records
      )
      idx <- rep(starts, each = size[[i]]) + seq_len(size[[i]]) - 1L
      res_fields[[i]] <- convert_bytes_to_array(
        bytes[idx],
        what = what[[i]],
        shape = NULL,
        size = size[[i]],
        endian = endian[[i]]
      )
    }

    # Final list-transpose
    res <- vector("list", n_records)
    for (j in seq_len(n_records)) {
      res[[j]] <- lapply(res_fields, `[[`, j)
    }
    dim(res) <- shape
    return(res)
  }

  if (is.na(endian)) {
    endian <- .Platform$endian
  }
  # FIXME: optimize this
  if (what != "unicode" && endian != .Platform$endian) {
    ind <- rep_len(rev(seq_len(size)), length(bytes)) +
      (seq_along(bytes) - 1L) %/% size * size
    bytes <- bytes[ind]
  }
  res <- .Call(
    C_type_convert,
    bytes,
    what,
    size,
    shape,
    endian,
    PACKAGE = "grumpy"
  )

  return(res)
}

py_dict_to_r_list <- function(x) {
  x |>
    # Logicals
    gsub("\\bNone\\b", "NA", x = _) |>
    gsub("\\bTrue\\b", "TRUE", x = _) |>
    gsub("\\bFalse\\b", "FALSE", x = _) |>
    # Trailing commas (before closing brackets/braces)
    gsub(",\\s*([]})])", "\\1", x = _) |>
    # Dict (single-quoted) keys to list names.
    #   Keys are composed of:
    #   - anything that is not the quote itself or a backslash,
    #   - a backslash followed by any character (to allow for escaped quotes).
    gsub("'(([^'\\\\]|\\\\.)*)'\\s*:", "'\\1' =", x = _) |>
    # Dict (double-quoted) keys to list names.
    gsub('"(([^"\\\\]|\\\\.)*)"\\s*:', '"\\1" =', x = _) |>
    # Tuples to c(...)
    gsub("(", "c(", x = _, fixed = TRUE) |>
    # Lists to list(...)
    gsub("[", "list(", x = _, fixed = TRUE) |>
    gsub("{", "list(", x = _, fixed = TRUE) |>
    # Closing brackets/braces
    gsub("]", ")", x = _, fixed = TRUE) |>
    gsub("}", ")", x = _, fixed = TRUE) |>
    # Evaluate the resulting R expression
    parse(text = _) |>
    eval()
}

Try the grumpy package in your browser

Any scripts or data that you put into this service are public.

grumpy documentation built on Nov. 16, 2026, 5:07 p.m.