R/parquet-column-types.R

In nanoparquet: Read and Write 'Parquet' Files

#' Map between R and Parquet data types
#'
#' Note that this function is now deprecated. Please use
#' [read_parquet_schema()] for files, and [infer_parquet_schema()] for
#' data frames.
#'
#' This function works two ways. It can map the R types of a data frame to
#' Parquet types, to see how [write_parquet()] would write out the data
#' frame. It can also map the types of a Parquet file to R types, to see
#' how [read_parquet()] would read the file into R.
#'
#' @param x Path to a Parquet file, or a data frame.
#' @param options Nanoparquet options, see [parquet_options()].
#' @return Data frame with columns:
#'   * `file_name`: file name.
#'   * `name`: column name.
#'   * `type`: (low level) Parquet data type.
#'   * `r_type`: the R type that corresponds to the Parquet type.
#'     Might be `NA` if [read_parquet()] cannot read this column. See
#'     [nanoparquet-types] for the type mapping rules.
#'   * `repetition_type`: whether the column in `REQUIRED` (cannot be
#'     `NA`) or `OPTIONAL` (may be `NA`). `REPEATED` columns are not
#'     currently supported by nanoparquet.
#'   * `logical_type`: Parquet logical type in a list column.
#'      An element has at least an entry called `type`, and potentially
#'      additional entries, e.g. `bit_width`, `is_signed`, etc.
#'
#' @seealso [read_parquet_metadata()] to read more metadata,
#'   [read_parquet_info()] for a very short summary.
#'   [read_parquet_schema()] for the complete Parquet schema.
#'   [read_parquet()], [write_parquet()], [nanoparquet-types].
#' @export

parquet_column_types <- function(x, options = parquet_options()) {
	warning(
		"`parquet_column_types()` is deprecated, please use ",
		"`read_parquet_schema()` or `parquet_schema()` instead."
  )
	if (is.character(x)) {
		parquet_column_types_file(x, options)
	} else if (is.data.frame(x)) {
		infer_parquet_schema(x, options)
	} else {
		stop("`x` must be a file name or a data frame in `parquet_column_types()`")
	}
}

parquet_column_types_file <- function(file, options) {
  mtd <- read_parquet_metadata(file)
	sch <- mtd$schema
	add_r_type_to_schema(mtd, sch, options)
}

add_r_type_to_schema <- function(mtd, sch, options, col_select = NULL) {
	kv <- mtd$file_meta_data$key_value_metadata[[1]]

	type_map <- c(
		BOOLEAN = "logical",
		INT32 = "integer",
		INT64 = "double",
		DOUBLE = "double",
		FLOAT = "double",
		INT96 = "POSIXct",
		FIXED_LEN_BYTE_ARRAY = "raw",
		BYTE_ARRAY = "raw"
	)

	sch$r_type <- unname(type_map[sch$type])

	sch$r_type[
		sch$type == "FIXED_LEN_BYTE_ARRAY" &
		sch$converted_type == "DECIMAL"] <- "double"
	sch$r_type[
		vapply(sch$logical_type, function(x) {
			!is.null(x$type) && x$type %in% c("STRING", "ENUM", "UUID")
		}, logical(1)) |
		sch$converted_type == "UTF8"] <- "character"

	# detected from Arrow schema
	if (options[["use_arrow_metadata"]]) {
		spec <- if ("ARROW:schema" %in% kv$key) {
			kv <- mtd$file_meta_data$key_value_metadata[[1]]
			arrow_find_special(
				kv$value[match("ARROW:schema", kv$key)],
				file
			)
		}
		if (length(spec$factor)) sch$r_type[spec$factor] <- "factor"
		if (length(spec$difftime)) sch$r_type[spec$difftime] <- "difftime"
	}

	# TODO: this is duplicated in the C++ code
	# our own conversions
	dates <- vapply(
		sch$logical_type,
		function(lt) !is.null(lt$type) && lt$type == "DATE",
		logical(1)
	) | sch$converted_type == "DATE"
	sch$r_type[dates] <- "Date"

	hmss <- vapply(
		sch$logical_type,
		function(lt) !is.null(lt$type) && lt$type == "TIME",
		logical(1)
	) | sch$converted_type == "TIME_MILLIS" | sch$converted_type == "TIME_MICROS"
	sch$r_type[hmss] <- "hms"

	poscts <- vapply(
		sch$logical_type,
		function(lt) !is.null(lt) && lt$type == "TIMESTAMP",
		logical(1)
	) | sch$converted_type == "TIMESTAMP_MICROS"
	sch$r_type[poscts] <- "POSIXct"

	cols <- c(
		"file_name",
		"name",
		"r_type",
		setdiff(colnames(sch), c("file_name", "name", "r_type"))
	)
	sch[, cols]
}