In jkennel/transducer: Reads RBR files

knitr::opts_chunk$set(echo = TRUE)

library(transducer)
library(data.table)
library(viridis)
library(knitr)
library(listviewer)

.rsk files

RBR stores data in .rsk files which are SQLite3 databases. This file contains additional meta data that is often lost when storing data as a csv.

Inputs

This packages requires the user to make two input files (.csv): - The location file which contains information on the transducer - The calibration file which contains start and end times for calibrations, and any manual measurement.

At the moment the structure is fairly restricted, but will likely become more flexible with time. The names of the columns are prescribed, however, the order doesn't matter.

File 1 (locations file)

locs_details <- data.table(`Field name` = c('file', 'elevation', 'is_baro', 'well', 'port', 'other'),
           Description = c('Path to file', 'Transducer elevation  (masl)', 'Is transducer in air?', 'The well name', 'Port id', 'Any other information'),
           `Data type` = c('character', 'numeric', 'logical', 'character', 'character', 'any'),
           Required = c(TRUE, TRUE, TRUE, TRUE, FALSE, FALSE))

kable(locs_details)

locations <- head(fread('/media/kennel/Data/phd/personnel/pat/syngenta/transducer_depth_path.csv'))
locations[, file_name := tail(strsplit(path, '/')[[1]], 1), by = 1:nrow(locations)]
loc_example <- locations[, list(file = file_name, well, port = id, elevation = (309.33 + 0.319)-depth, is_baro = baro)]
locations <- locations[, list(file = path, well, port = id, elevation = (309.33 + 0.319)-depth, is_baro = baro)]

kable(head(loc_example))

File 2 (Manual measurement/Calibration file)

man_details <- data.table(`Field name` = c('start', 'end', 'value_manual'),
           Description = c('Start (2014-12-08 14:50:00)', 'End (2014-12-08 14:55:00)', 'Manual water level elevation (masl)'),
           `Data type` = c('character', 'character', 'numeric'),
           Required = c(TRUE, TRUE, TRUE))

kable(man_details)

manual_wl <- data.table(start = c(as.POSIXct('2014-12-08 14:50:00', tz = 'UTC'),
                                  as.POSIXct('2014-12-08 14:52:00', tz = 'UTC')),
                        end   = c(as.POSIXct('2014-12-08 14:52:00', tz = 'UTC'),
                                  as.POSIXct('2014-12-08 14:54:00', tz = 'UTC')),
                        well  = c('UW-1', 'UW-2'),
                        value_manual = c((309.33 + 0.319) - 17.45) + c(0, 0))
kable(manual_wl)


locations[6, well := 'UW-2']
locations[1, well := 'UW-2']

Reading files

Get start and end times

rbr_start_end(copy(locations))

We can use the locations file to read in data. There is a lot of useful information in the .rsk file which is added to the location file information. The result is stored in a nested data.table with the following columns:

# Read data

wl <- read_rbr(locations, 
               start = as.POSIXct('2015-03-10 09:00:00', tz = 'UTC'), 
               end   = as.POSIXct('2015-03-10 16:00:00', tz = 'UTC'))

wl_example <- data.table(`Field name` = names(wl),
           Description = c('The full path to file',
                           'The file_name without the path',
                           'Model of the rbr sensor',
                           'Sensor serial number',
                           'The port name',
                           'Is the transducer in air?',
                           'The transducer sensor elevation',
                           'The channel name from the database',
                           'Channel type',
                           'The version of ruskin that created the file',
                           'Time interval in milliseconds',
                           'Number of measurements',
                           'Units of the measurement',
                           'The id in the *.rsk file',
                           'data.table with datetime and value',
                           'data.table of calibration values',
                           'other information: file_id',
                           'other information: well name'),
           `Data type` = c('character', 'character', 'character',
                           'character', 'character', 'logical', 
                           'numeric', 'character','character','character',
                           'integer','integer', 'character', 'integer',
                           'data.table', 'data.table','integer', 'character'))


kable(wl_example)

Manual Measurement calibration

The calibration file is used to get the differences between the measured transducer values and a water level measurement. First we calculate the shift.

# calculate the shifts
shifts <- compare_manual(locations, manual_wl)

Then we apply the shift to the dataset.

# apply the shifts
wl <- adjust_value(wl, shifts)

If we want to pull out the data along with some other key columns we can unnest. Because there is an elevation column in the location table we also get an estimated water level in meters. The barometric values are also converted to an equivalent water height in meters.

wl[, (data[[1]]), by = list(well, port, serial)]
wl[, (calibration[[1]]), by = list(well, port, serial)]

Preview

Hydrograph

# Horizontal profile
hydrograph(wl, var_type = 'pressure', plot_col = 'value_adj')

Vertical Profile

With water level adjustment

# Vertical profile
# We will read in a smaller subset
# Generate weekly sequence of dates
times <- seq.int(as.numeric(as.POSIXct('2014-12-15', tz = 'UTC')),
                 as.numeric(as.POSIXct('2015-04-01', tz = 'UTC')),
                 by = 86400*7)

# read data
wl <- read_rbr(locations, times = times)
# apply shifts
wl <- adjust_value(wl, shifts)
# plot
vertigraph(wl[!serial %in% c(78180, 78189)], 
           var_type = 'pressure', plot_col = 'value_adj')

With baro adjustment

air <- data.table(start = as.POSIXct('2014-12-08 00:02:00', tz = 'UTC'),
                  end   = as.POSIXct('2014-12-08 00:05:00', tz = 'UTC'),
                  well  = c('UW-1', 'UW-2'))

shifts <- compare_air(locations, air)

# apply shifts
wl <- adjust_value(wl, shifts)
# plot
vertigraph(wl[!serial %in% c(78180, 78189)], 
           var_type = 'pressure', 
           plot_col = 'value_adj')

No adjustment

vertigraph(wl[!serial %in% c(78180, 78189)], 
           var_type = 'pressure', plot_col = 'value_wl')

jkennel/transducer documentation built on Feb. 1, 2024, 9:45 a.m.