R/codec.R

#' Storage descriptors
#'
#' A storage descriptor says how the values of an image sit in a stream of
#' bytes: their type, byte order and position, any scaling applied to them, and
#' the order of the image's axes in the stream. It deliberately says nothing
#' about any particular file format. A format package parses a header into a
#' descriptor, and [readImageData()] and [writeImageData()] do the rest.
#'
#' Types are named by size and signedness: `"int8"`, `"uint8"`, `"int16"`,
#' `"uint16"`, `"int32"`, `"uint32"`, `"int64"`, `"float32"`, `"float64"`,
#' `"complex64"` and `"complex128"`, plus `"rgb24"` and `"rgba32"` for colour
#' and `"bit"` for one bit per value. The C-style names `"char"`, `"int"`,
#' `"float"` and `"double"` are accepted too, as `"int8"`, `"int32"`,
#' `"float32"` and `"float64"` respectively. C leaves the signedness of `char`
#' to the platform, so it is fixed as signed here; use `"uint8"` for unsigned
#' bytes.
#'
#' When read, integer types of up to 32 bits become R integers unless scaled,
#' and wider integers, floating-point types and scaled values become doubles.
#' 64-bit integers beyond 2^53 cannot be held exactly as doubles and are
#' refused. Colour values become integers from 0 to 255, with an extra
#' trailing dimension of three or four channels. Bits become logical values.
#' Scaling is not applied to complex or colour data.
#'
#' @param type The stored data type, as above.
#' @param endian The byte order of the stream: `"little"` or `"big"`.
#' @param offset The position of the first value, in bytes from the start of
#'   the stream.
#' @param slope,intercept Scaling, as `value = stored * slope + intercept`.
#'   When reading, `NULL` means no scaling. When writing, `NULL` means that
#'   a scaling is chosen to use the whole range of an integer type, if the
#'   data need one.
#' @param layout How the image's axes map onto the order of values in the
#'   stream, as described for [storageLayout()]. `NULL` means the stream
#'   holds the image in its own axis order, first axis fastest.
#' @param bitOrder For `"bit"` data, whether the first value of each byte is
#'   its least or most significant bit.
#' @return An object of S7 class `storageDescriptor`.
#' @examples
#' storageDescriptor("int16", endian = "big", offset = 352)
#' @export
storageDescriptor <- S7::new_class("storageDescriptor",
    properties = list(
        type = S7::class_character,
        endian = S7::class_character,
        offset = S7::class_double,
        slope = S7::class_double,
        intercept = S7::class_double,
        layout = S7::class_integer,
        bitOrder = S7::class_character
    ),
    validator = function (self) {
        if (length(self@type) != 1L || !self@type %in% names(codecTypeSizes))
            return(paste0("@type must be one of ", paste(names(codecTypeSizes), collapse = ", ")))
        if (length(self@endian) != 1L || !self@endian %in% c("little", "big"))
            return("@endian must be \"little\" or \"big\"")
        if (length(self@offset) != 1L || is.na(self@offset) || self@offset < 0)
            return("@offset must be a single non-negative value")
        if (length(self@slope) != 1L || length(self@intercept) != 1L)
            return("@slope and @intercept must each be a single value, which may be NA")
        if (isTRUE(self@slope == 0) || isTRUE(!is.finite(self@slope) && !is.na(self@slope)))
            return("@slope must be finite and non-zero")
        if (length(self@layout) > 0L)
        {
            problem <- checkLayout(self@layout, length(self@layout))
            if (!is.null(problem))
                return(problem)
        }
        if (length(self@bitOrder) != 1L || !self@bitOrder %in% c("lsb", "msb"))
            return("@bitOrder must be \"lsb\" or \"msb\"")
        NULL
    },
    constructor = function (type, endian = "little", offset = 0, slope = NULL, intercept = NULL,
                            layout = NULL, bitOrder = "lsb")
    {
        type <- as.character(type)
        if (length(type) == 1L && type %in% names(codecTypeAliases))
            type <- codecTypeAliases[[type]]
        S7::new_object(S7::S7_object(),
            type = type,
            endian = as.character(endian),
            offset = as.double(offset),
            slope = as.double(slope %||% NA_real_),
            intercept = as.double(intercept %||% NA_real_),
            layout = as.integer(layout %||% integer(0)),
            bitOrder = as.character(bitOrder))
    })

S7::method(print, storageDescriptor) <- function (x, ...)
{
    cat(sprintf("Storage descriptor: %s, %s-endian, from byte %s\n", x@type, x@endian, format(x@offset)))
    if (!is.na(x@slope) || !is.na(x@intercept))
        cat(sprintf("  Scaling: value = stored * %g + %g\n", x@slope %|NA|% 1, x@intercept %|NA|% 0))
    if (length(x@layout) > 0L && !isIdentityLayout(x@layout))
        cat(sprintf("  Layout : %s\n", paste(x@layout, collapse = ", ")))
    invisible(x)
}

`%|NA|%` <- function (x, y) if (is.na(x)) y else x

codecTypeSizes <- c(int8 = 1, uint8 = 1, int16 = 2, uint16 = 2, int32 = 4, uint32 = 4, int64 = 8,
                    float32 = 4, float64 = 8, complex64 = 8, complex128 = 16, rgb24 = 3, rgba32 = 4,
                    bit = 1/8)

codecTypeAliases <- c(char = "int8", int = "int32", float = "float32", double = "float64")

## Colour is stored as bytes with the channel varying fastest, which is an
## extra storage axis of rank one in front of the image's own. The image sees
## the channels as a trailing value dimension
colourChannels <- c(rgb24 = 3L, rgba32 = 4L)

#' Reading and writing image data
#'
#' `readImageData()` reads the values of an image from a connection, as
#' described by a [storageDescriptor()], and returns an image.
#' `writeImageData()` writes them. Neither knows anything about file formats,
#' which are the business of packages built on this one: a format package
#' parses any header or metadata, builds a descriptor and a geometry, and calls
#' these to handle the data.
#'
#' Bytes are read through R connections, so compressed streams need nothing
#' special: [gzfile()] reads gzipped and plain files alike. They are read in
#' chunks, so the only large allocation is the result itself. A gzipped stream
#' cannot be read backwards, so volumes are always read in the order they are
#' stored, whatever order they are asked for in.
#'
#' The image comes back dense, packed or sparse, according to the `as`
#' argument. The default, `"auto"`, gives a sparse image if a `mask` is given;
#' otherwise a packed image if the stored type is one a [packedImage] can hold,
#' keeping the file's narrow representation and scaling; and otherwise a dense
#' image. The class returned therefore depends on the data, so code that relies
#' on a specific class should ask for it.
#'
#' A packed image keeps the stream's order, with the layout recording how its
#' axes map onto it, so no data is moved; see [reorient()]. A dense image is
#' always in its own axis order, so a stream in any other order is rearranged
#' as it is read.
#'
#' A sparse image keeps exactly the locations selected by `mask`, if one is
#' given, and otherwise those where any value is non-zero. Only volume-ordered
#' data can be read straight into sparse form, without the dense array ever
#' existing; other layouts, and bit data, are read densely first.
#'
#' @param con A connection, or the path to a file. A connection must be open
#'   for binary reading or writing. If its position can be queried, it is
#'   moved forwards to the descriptor's offset, or when writing, padded with
#'   zeros up to it; otherwise it is taken to be already there. A path is
#'   opened, read or written from the start, and closed.
#' @param descriptor A [storageDescriptor()].
#' @param dims The full dimensions of the image, in its own axis order.
#' @param geometry An [imageGeometry][geometry], or an image to take one from,
#'   for the spatial grid. By default the leading three dimensions (or all of
#'   them, if fewer) are spatial, with unit voxels at the origin.
#' @param as The representation wanted: `"auto"`, `"dense"`, `"sparse"` or
#'   `"packed"`.
#' @param volumes Optionally, which volumes to read, as one-based indices into
#'   the values held at each location, taken together. The result then has the
#'   spatial dimensions followed by one dimension over the volumes read, in the
#'   order given. Only data stored a volume at a time can be read this way.
#' @param mask Optionally, the spatial locations to keep, in any form
#'   [asMaskVector()] accepts. Only meaningful for a sparse result.
#' @param image An image or array to write.
#' @return `readImageData()` returns an image. `writeImageData()` returns,
#'   invisibly, the descriptor as used, with any scaling it chose filled in,
#'   which is what a format package needs to write its header.
#' @examples
#' image <- denseImage(array(1:24, c(2, 3, 4)))
#' descriptor <- storageDescriptor("int16", endian = "big")
#' file <- tempfile()
#' writeImageData(image, file, descriptor)
#' readImageData(file, descriptor, dim(image))
#' @export
readImageData <- function (con, descriptor, dims, geometry = NULL, as = c("auto", "dense", "sparse", "packed"),
                           volumes = NULL, mask = NULL)
{
    if (!S7::S7_inherits(descriptor, storageDescriptor))
        stop("A storageDescriptor is needed to describe the data")
    as <- match.arg(as)
    dims <- as.integer(dims)
    geometry <- resolveGeometry(dims, from = geometry)
    nSpatial <- length(geometry@dims)
    type <- descriptor@type
    slope <- descriptor@slope %|NA|% 1
    intercept <- descriptor@intercept %|NA|% 0
    layout <- if (length(descriptor@layout) == 0L) seq_along(dims) else descriptor@layout
    if (length(layout) != length(dims))
        stop("The descriptor's layout has ", length(layout), " entries, but the image has ", length(dims), " dimensions")

    ## Colour becomes bytes with an extra, fastest-varying storage axis, seen by
    ## the image as a trailing dimension
    streamDims <- dims
    if (type %in% names(colourChannels))
    {
        streamDims <- c(dims, colourChannels[[type]])
        layout <- c(sign(layout) * (abs(layout) + 1L), 1L)
        type <- "uint8"
        slope <- 1
        intercept <- 0
    }

    if (as == "auto")
        as <- if (!is.null(mask)) "sparse" else if (descriptor@type %in% storageTypes) "packed" else "dense"
    if (!is.null(mask) && as != "sparse")
        stop("A mask is only used when reading to a sparse image")
    if (as == "packed" && !type %in% storageTypes)
        stop("Data of type ", descriptor@type, " cannot be kept packed; read them densely instead")

    ## Whether values are stored a volume at a time: the spatial axes first, in
    ## any order and direction among themselves, then the rest in order
    blockOrdered <- all(abs(layout[seq_len(nSpatial)]) <= nSpatial) &&
                    isIdentityLayout(layout[-seq_len(nSpatial)] - nSpatial)
    nVolumes <- prod(streamDims[-seq_len(nSpatial)])

    blocks <- NULL
    outDims <- streamDims
    if (!is.null(volumes))
    {
        volumes <- as.integer(volumes)
        if (length(volumes) == 0L || anyNA(volumes) || any(volumes < 1L) || any(volumes > nVolumes))
            stop("Volumes must be between 1 and ", nVolumes)
        if (!blockOrdered)
            stop("Volumes can only be selected from data stored a volume at a time")
        blocks <- as.double(volumes - 1L)
        outDims <- c(streamDims[seq_len(nSpatial)], length(volumes))
    }

    ## Sparse results are built a volume at a time, which needs volume order.
    ## Anything else is read densely and converted
    viaDense <- (as == "sparse" && (!blockOrdered || type == "bit"))
    selected <- if (is.null(mask)) NULL else asMaskVector(mask, geometry@dims)

    if (is.character(con))
    {
        con <- gzfile(con, "rb")
        on.exit(close(con))
    }
    reader <- streamReader(con)
    reader$skip(descriptor@offset - reader$position())

    ## Reads that keep the stream's order carry its layout with them, reduced
    ## to the spatial part plus one volume axis when volumes were selected
    keptLayout <- if (is.null(blocks)) layout else c(layout[seq_len(nSpatial)], nSpatial + 1L)
    if (as == "sparse" && !viaDense && is.null(blocks))
        keptLayout <- c(layout[seq_len(nSpatial)], seq_len(length(outDims) - nSpatial) + nSpatial)

    result <- decodeImage(reader$read, reader$skip, type, descriptor@endian != .Platform$endian,
                          slope, intercept, streamDims, if (isIdentityLayout(layout)) NULL else layout,
                          nSpatial, if (viaDense) "dense" else as, blocks,
                          if (viaDense) NULL else selected, descriptor@bitOrder == "msb")

    if (as == "packed")
        return(packedImage(values = result, storageType = type, dims = outDims, slope = slope,
                           intercept = intercept, geometry = geometry, layout = keptLayout))

    if (as == "sparse" && !viaDense)
    {
        ## Sparse storage has one row per location, one column per value held
        values <- result$values
        values <- if (length(outDims) > nSpatial) values else as.vector(values)
        return(sparseImage(mask = result$mask, values = values, dim = outDims, geometry = geometry,
                           layout = keptLayout))
    }

    dim(result) <- outDims
    if (as != "sparse")
        return(denseImage(result, geometry = geometry))
    if (is.null(selected))
        return(asSparse(denseImage(result, geometry = geometry)))

    ## A sparse image keeping exactly the locations selected
    nLocations <- prod(outDims[seq_len(nSpatial)])
    dim(result) <- c(nLocations, length(result) %/% max(nLocations, 1L))
    sparseImage(mask = selected, values = t(result[selected, , drop = FALSE]), dim = outDims,
                geometry = geometry)
}

## Closures over a connection for the compiled decoder. Plain files are
## skipped through by seeking; anything else, including compressed streams,
## is read and discarded, which is the only way forward through a gzip stream
streamReader <- function (con)
{
    canSeek <- isTRUE(tryCatch(isSeekable(con), error = function (e) FALSE)) &&
               identical(summary(con)$class, "file")

    read <- function (n) readBin(con, "raw", n)

    skip <- function (n)
    {
        if (n < 0)
            stop("The connection is already past the start of the data")
        if (n == 0)
            return(invisible(NULL))
        if (canSeek)
            seek(con, n, origin = "current")
        else
        {
            while (n > 0)
            {
                chunk <- min(n, 2^26)
                if (length(readBin(con, "raw", chunk)) < chunk)
                    stop("Unexpected end of data while skipping forward")
                n <- n - chunk
            }
        }
        invisible(NULL)
    }

    position <- function ()
    {
        where <- tryCatch(seek(con), error = function (e) NA_real_)
        if (is.na(where)) 0 else where
    }

    list(read = read, skip = skip, position = position)
}

#' @rdname readImageData
#' @export
writeImageData <- function (image, con, descriptor)
{
    if (!S7::S7_inherits(descriptor, storageDescriptor))
        stop("A storageDescriptor is needed to describe the data")

    type <- descriptor@type
    dims <- dim(image) %||% length(image)
    swap <- (descriptor@endian != .Platform$endian)

    if (type %in% names(colourChannels))
    {
        ## The channels are the image's last dimension, but the stream's
        ## fastest-varying axis; the descriptor's layout covers the rest
        channels <- colourChannels[[type]]
        if (utils::tail(dims, 1L) != channels)
            stop("Colour data of type ", type, " need a last dimension of ", channels)
        layout <- if (length(descriptor@layout) == 0L) seq_len(length(dims) - 1L) else descriptor@layout
        if (length(layout) != length(dims) - 1L)
            stop("The descriptor's layout has ", length(layout), " entries, but the image has ",
                 length(dims) - 1L, " dimensions besides its colour channels")
        layout <- c(sign(layout) * (abs(layout) + 1L), 1L)
        type <- "uint8"
        descriptor@slope <- descriptor@intercept <- NA_real_
    }
    else
        layout <- if (length(descriptor@layout) == 0L) seq_along(dims) else descriptor@layout

    if (length(layout) != length(dims))
        stop("The descriptor's layout has ", length(layout), " entries, but the image has ", length(dims), " dimensions")

    ## Packed data already in the form wanted are written as they stand
    if (isPackedImage(image) && image@storageType == type &&
        (is.na(descriptor@slope) || isTRUE(all.equal(descriptor@slope, image@slope))) &&
        (is.na(descriptor@intercept) || isTRUE(all.equal(descriptor@intercept, image@intercept))) &&
        identical(as.integer(layout), image@layout))
    {
        descriptor@slope <- image@slope
        descriptor@intercept <- image@intercept
        bytes <- if (swap) swapBytes(image@values, storageTypeSize[[type]]) else image@values
    }
    else
    {
        values <- as.array(image)
        if (is.na(descriptor@slope) || is.na(descriptor@intercept))
        {
            ## The narrow integer types get a scaling that uses their whole
            ## range when the data need it, as for asPacked(); everything else
            ## is written as it stands
            chosen <- list(slope = 1, intercept = 0)
            if (type %in% setdiff(storageTypes, "float32") && typeof(values) %in% c("logical", "integer", "double"))
            {
                summary <- valueRange(values)
                if (is.finite(summary$low) && is.finite(summary$high))
                    chosen <- calibrateStorage(type, summary$low, summary$high, summary$integral)
            }
            descriptor@slope <- descriptor@slope %|NA|% chosen$slope
            descriptor@intercept <- descriptor@intercept %|NA|% chosen$intercept
        }
        bytes <- encodeImage(values, type, swap, descriptor@slope, descriptor@intercept, dims,
                             if (isIdentityLayout(layout)) NULL else layout, descriptor@bitOrder == "msb")
    }

    if (is.character(con))
    {
        con <- if (grepl("\\.gz$", con, ignore.case = TRUE)) gzfile(con, "wb") else file(con, "wb")
        on.exit(close(con))
    }
    position <- tryCatch(seek(con), error = function (e) NA_real_)
    if (!is.na(position))
    {
        if (position > descriptor@offset)
            stop("The connection is already past the offset the data should start at")
        if (position < descriptor@offset)
            writeBin(raw(descriptor@offset - position), con)
    }
    writeBin(bytes, con)

    if (descriptor@type %in% names(colourChannels))
        descriptor@slope <- descriptor@intercept <- NA_real_
    invisible(descriptor)
}

Try the imply package in your browser

Any scripts or data that you put into this service are public.

imply documentation built on Nov. 20, 2026, 5:12 p.m.