In rappster/laker: Local data lake framework

knitr::opts_chunk$set(
    collapse = TRUE,
    comment = "#>"
)

# library(dbt.def.core)
devtools::load_all(here::here())
# box::use(dl = R/data_lake)

Resources

• Data lake layers: https://lingarogroup.com/blog/data-lake-architecture/

Ingest (from layer_01 = raw to layer_02 = tidy)

pops::get_options()

layer_ingest(
    constructor = data_tableau_global_superstore
)

Read from layer_02 layer

superstore <- layer_read(
    constructor = data_tableau_global_superstore,
    layer = valid_data_layers("02")
)

superstore %>% class()

Curate (from layer_02 to layer_03)

superstory_cura <- superstore %>% layer_curate()

Read from 03_curated layer

superstore_cura <- layer_read(
    constructor = data_tableau_global_superstore,
    layer = valid_data_layers("03")
)

Inspect

Overview

superstore_cura %>% dplyr::glimpse()

Data table

superstore_cura %>% 
    dplyr::slice(1:1000) %>% 
    DT::datatable()

superstore_cura %>% 
    dplyr::slice(1:1000) %>% 
    datatable_2()

library(synthpop)
superstore_cura_syn <- superstore_cura %>% 
    dplyr::mutate(
        order_date = order_date %>% as.character(),
        ship_date = ship_date %>% as.character()
    ) %>% 
    dplyr::sample_n(100) %>% 
    synthpop::syn(maxfaclevels = 26000)
iris_syn %>% attributes()
iris_syn_tbl <- iris_syn$syn %>% tibble::as_tibble()

Analysis

Frequency tables

superstore_cura %>% 
    stats_freq_table(segment) %>% 
    datatable_2()

# Nicer names
freq_tab <- superstore_cura %>% stats_freq_table(segment)
names(freq_tab)[2:length(names(freq_tab))] <- c("Anzahl", "Prozent", "Prozent inkl. N/A")

freq_tab %>% datatable_2()

Plots

Barplot

superstore_cura %>% 
    plot_bar_plotly(
        col_x = market,
        col_y = sales,
        title = "Verteilung der Märkte"
    )

Histogram

superstore_cura %>% plot_histogram_plotly(
    col_x = sales
)

rappster/laker documentation built on Aug. 15, 2022, 10:13 p.m.