In leppott/ContDataQC: Quality Control (QC) of Continous Monitoring Data

Helper code for creating library.

[e-mail] 2017-09-26

Package

Package related code.

Vignette

1. Need to run code below to add data to "vignette" directory so it will work.

# 1. change wd to vignettes
# setwd(file.path("C:","Users","Erik.Leppo","OneDrive - Tetra Tech, Inc"
#                 ,"MyDocs_OneDrive","GitHub","ContDataQC","vignettes"))
setwd("vignettes")
#
# 2. Add data for vignette examples
# Parameters
Selection.Operation <- c("GetGageData","QCRaw", "Aggregate", "SummaryStats")
Selection.Type      <- c("Air","Water","AW","Gage","AWG","AG","WG")
Selection.SUB <- c("Data1_RAW","Data2_QC","Data3_Aggregated","Data4_Stats")
myDir.BASE <- getwd()
#
# Create data directories
myDir.create <- paste0("./",Selection.SUB[1])
  ifelse(dir.exists(myDir.create)==FALSE,dir.create(myDir.create),"Directory already exists")
myDir.create <- paste0("./",Selection.SUB[2])
  ifelse(dir.exists(myDir.create)==FALSE,dir.create(myDir.create),"Directory already exists")
myDir.create <- paste0("./",Selection.SUB[3])
  ifelse(dir.exists(myDir.create)==FALSE,dir.create(myDir.create),"Directory already exists")
myDir.create <- paste0("./",Selection.SUB[4])
  ifelse(dir.exists(myDir.create)==FALSE,dir.create(myDir.create),"Directory already exists")
#
# Save example data (assumes directory ./Data1_RAW/ exists)
myData <- data_raw_test2_AW_20130426_20130725
  write.csv(myData,paste0("./",Selection.SUB[1],"/test2_AW_20130426_20130725.csv"))
myData <- data_raw_test2_AW_20130725_20131015
  write.csv(myData,paste0("./",Selection.SUB[1],"/test2_AW_20130725_20131015.csv"))
myData <- data_raw_test2_AW_20140901_20140930
  write.csv(myData,paste0("./",Selection.SUB[1],"/test2_AW_20140901_20140930.csv"))
myData <- data_raw_test4_AW_20160418_20160726
  write.csv(myData,paste0("./",Selection.SUB[1],"/test4_AW_20160418_20160726.csv"))
myFile <- "config.TZ.Central.R"
  file.copy(file.path(path.package("ContDataQC"),"extdata",myFile)
            ,file.path(getwd(),Selection.SUB[1],myFile))

1. Knit the vignette in the vignette folder to ensure all of the code works and is properly displayed. This can take 2 to 5 min.

2. Use the code below to "build" the vignette and then use it in the pkg.

# generate Vignette
library(ContDataQC)
library(devtools)
devtools::build_vignettes()

# create vignette folder and default file
#devtools::use_vignette("ContDataQC_Vignette")

devtools::build() more useful and build_vignettes()

Create Package

Use the code below to create the package. Assumes you are in the development package directory

# Set working directory
myLibrary <- "ContDataQC"
dir_base <- "C:/Users/Erik.Leppo/Documents/GitHub"
#setwd(file.path(dir_base, myLibrary))


# Remove all files in "Results" folder
# Triggered here so can run different files
path_shiny <- file.path("inst", "shiny-examples", "ContDataQC")
path_results <- file.path(path_shiny, "data")
fn_results <- list.files(path_results
                         , full.names = TRUE
                         , include.dirs = FALSE
                         , recursive = TRUE)
file.remove(fn_results) # ok if no files
# Copy file to ensure directory not empty
fn_copy    <- "remove.txt"
path_from  <- file.path(path_shiny, "external", fn_copy)
path_to    <- file.path(path_shiny, "data", fn_copy)
file.copy(path_from, path_to)
# also HOBO folder
file.remove(list.files(file.path(path_shiny, "HOBO"),
                       full.names = TRUE,
                       include.dirs = TRUE,
                       recursive = TRUE))
file.copy(path_from, file.path(path_shiny, "HOBO", fn_copy))

# NEWS
# Render then Copy NEWS so picked up in help
rmarkdown::render("NEWS.rmd", "all")
file.copy("NEWS.md", "NEWS", overwrite = TRUE)
file.remove("NEWS.html")
#file.remove("NEWS.md") # keep for pkgdown CHANGE LOG
#
# Load Library
library(devtools)
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Create Package
# create(myLibrary)
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Document, Install, and Reload Library
## Generate Documentation
devtools::document()
## Install New Package (locally)
setwd("..") # return to root directory first
devtools::install(myLibrary
                  , build_vignettes = TRUE
                  , quick = FALSE
                  , reload = TRUE)

## Reload library
library(myLibrary,character.only = TRUE)
# change wd back to package
setwd(file.path(dir_base, myLibrary))
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

After creating the package reload it after restarting R within RStudio (Ctrl + Shift + F10).

# Restart R within RStudio:  Ctrl + Shift + F10
pkg <- "ContDataQC"
library(pkg, character.only = TRUE)
help(package=(pkg))
shell.exec(tempdir())
#
?PeriodStats
#?CompSiteCDF
??ContDataQC
?fun.Export.IHA
?Export.StreamThermal  

Package website Use pkgdown to create site.

https://www.r-bloggers.com/tvthemes-1-1-0-is-on-cran-creating-a-pkgdown-website-gravity-falls-palette-and-more/

#library(pkgdown)

usethis::use_pkgdown()
pkgdown::build_site()

code perfomance

time each part of code

# Packages
library(profvis)

# Run example code
## Set up
# Examples of each operation

# 00. Set up
# Parameters
Selection.Operation <- c("GetGageData"
                         , "QCRaw"
                         , "Aggregate"
                         , "SummaryStats")
Selection.Type      <- c("Air","Water","AW","Gage","AWG","AG","WG")
Selection.SUB <- c("Data0_Original"
                   , "Data1_RAW"
                   , "Data2_QC"
                   , "Data3_Aggregated"
                   , "Data4_Stats")
(myDir.BASE <- tempdir()) # create and print temp directory for example data

# Create data directories
myDir.create <- file.path(myDir.BASE, Selection.SUB[1])
  ifelse(dir.exists(myDir.create) == FALSE
         , dir.create(myDir.create)
         , "Directory already exists")
myDir.create <- file.path(myDir.BASE, Selection.SUB[2])
  ifelse(dir.exists(myDir.create) == FALSE
         , dir.create(myDir.create)
         , "Directory already exists")
myDir.create <- file.path(myDir.BASE, Selection.SUB[3])
  ifelse(dir.exists(myDir.create) == FALSE
         , dir.create(myDir.create)
         , "Directory already exists")
myDir.create <- file.path(myDir.BASE, Selection.SUB[4])
  ifelse(dir.exists(myDir.create) == FALSE
         , dir.create(myDir.create)
         , "Directory already exists")
myDir.create <- file.path(myDir.BASE, Selection.SUB[5])
  ifelse(dir.exists(myDir.create) == FALSE
         , dir.create(myDir.create)
         , "Directory already exists")

# Save example data (assumes myDir.BASE directory exists)
myData <- data_raw_test2_AW_20130426_20130725
  write.csv(myData, file.path(myDir.BASE
                              , Selection.SUB[2]
                              , "test2_AW_20130426_20130725.csv"))
myData <- data_raw_test2_AW_20130725_20131015
  write.csv(myData, file.path(myDir.BASE
                              , Selection.SUB[2]
                              , "test2_AW_20130725_20131015.csv"))
myData <- data_raw_test2_AW_20140901_20140930
  write.csv(myData, file.path(myDir.BASE
                              , Selection.SUB[2]
                              , "test2_AW_20140901_20140930.csv"))
myData <- data_raw_test4_AW_20160418_20160726
  write.csv(myData, file.path(myDir.BASE
                              , Selection.SUB[2]
                              , "test4_AW_20160418_20160726.csv"))
myFile <- "config.TZ.Central.R"
  file.copy(file.path(path.package("ContDataQC"), "extdata", myFile)
            , file.path(myDir.BASE, Selection.SUB[2], myFile))

myData.Operation       <- "QCRaw" #Selection.Operation[2]
myData.SiteID          <- "test2"
myData.Type            <- Selection.Type[3] #"AW"
myData.DateRange.Start <- "2013-01-01"
myData.DateRange.End   <- "2014-12-31"
myDir.import           <- file.path(myDir.BASE, Selection.SUB[2]) #"Data1_RAW"
myDir.export           <- file.path(myDir.BASE, Selection.SUB[3]) #"Data2_QC"
myReport.format        <- "docx"


# Save
p <- profvis({
  ContDataQC(myData.Operation
           , myData.SiteID
           , myData.Type
           , myData.DateRange.Start
           , myData.DateRange.End
           , myDir.import
           , myDir.export
           , fun.myReport.format = myReport.format)
})

# Save to HTML
htmlwidgets::saveWidget(p, file.path(tempdir(), "profile.html"))

# Open in browser from R
browseURL(file.path(tempdir(), "profile.html"))

Testing of configuration and RMD files.

library(ContDataQC)
# Parameters
Selection.Operation <- c("GetGageData","QCRaw", "Aggregate", "SummaryStats")
Selection.Type      <- c("Air","Water","AW","Gage","AWG","AG","WG")
Selection.SUB <- c("Data1_RAW","Data2_QC","Data3_Aggregated","Data4_Stats")
myDir.BASE <- getwd()
#
# Aggregate Data
myData.Operation <- "Aggregate" #Selection.Operation[3]
myData.SiteID    <- "test2"
myData.Type      <- Selection.Type[3] #"AW"
myData.DateRange.Start  <- "2013-01-01"
myData.DateRange.End    <- "2014-12-31"
myDir.import <- file.path(myDir.BASE,Selection.SUB[2]) #"Data2_QC"
myDir.export <- file.path(myDir.BASE,Selection.SUB[3]) #"Data3_Aggregated"
myReport.format <- "docx"
#
# set config file with RMD directory as getwd()
myConfig            <- file.path(getwd(),Selection.SUB[2],"config.RMDdir.R")
# run function
ContDataQC(myData.Operation, myData.SiteID, myData.Type, myData.DateRange.Start
           , myData.DateRange.End, myDir.import, myDir.export
           , fun.myReport.format=myReport.format, fun.myConfig = myConfig)

Including Data

All data should have already been created.

Check

# Check for errors (or press Cmd + Shift + E in RStudio)
#http://r-pkgs.had.co.nz/check.html
devtools::check()

If use RStudio, Build - Check Package, the output is easier to read.

Test

Need to use testthat but not to that stage yet.

RStudio, Build - Test Package.

CI, Badges, Code Coverage, etc

• Continuous Integration with GitHub Actions (TravisCI shut down Dec 2020)
• Code Quality (CodeFactor)
  • CodeFactor.io
    • GitHub login
• Code Coverage (testing)
  • Codecov.io
    • GitHub login
    • After set up CI with GitHub Actions
• Package website (pkgdown and GitHub Action)
  • branch gh-pages, auto updates with each commit
  • Update in settings the location
• Badges; Maintained, Life Cycle, License, Issues, etc
  • https://github.com/GuangchuangYu/badger
    • Commands to create badges

badger package creates the text for pkg readme file.

Also https://shields.io/
https://github.com/badges/shields Can make some badges dynamic, static ones below

issues

release

downloads

pkg_GH <- "leppott/ContDataQC"
pkg_lic <- "MIT"

# GitHub Action, CI
usethis::use_github_actions()
usethis::use_github_actions_badge()
# badger::badge_github_actions(pkg_GH)

# code coverage
usethis::use_github_action("test-coverage")

# GitHub Action, pkgdown
usethis::use_github_action("pkgdown")

# Badges
# lifecycle
badger::badge_lifecycle("stable", "green") # stable/green, maturing/dormant/blue
# maintained
# license
badger::badge_license()
# code quality
badger::badge_codefactor(pkg_GH)
# code testing coverage
badger::badge_codecov(pkg_GH)
# cmd check
badger::badge_github_actions(pkg_GH)

Other Stuff

#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Upload to Github via GitHub Desktop utility
# 0. download from web via "clone or download" via "Open in Desktop" (GitHub Desktop) if not already in GitHub Desktop
# 1. Make changes in download/clone folder. (done above)
# 3. Open GH Desktop commit changes then sync.
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# install from GitHub (via devtools)
myLibrary <- "ContDataQC"
devtools::install_github(paste0("leppott/",myLibrary))
#



#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# remove installed packages (if needed for troubleshooting)
search() # find
#detach(3) # remove by number
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~


# to build package
#https://thepoliticalmethodologist.com/2014/08/14/building-and-maintaining-r-packages-with-devtools-and-roxygen2/
# To build the package as a compressed file in your working directory, run build(current.code, path=getwd()).

# to save internal data for examples
# example
#http://r-pkgs.had.co.nz/data.html#data-sysdata
# have to be at root directory (above package)
#devtools::use_data(NV.predictors,NV.bugs,pkg="MMIcalcNV",internal=TRUE,overwrite=TRUE)
## verify with data()

# To save RMD files
# http://stackoverflow.com/questions/30377213/how-to-include-rmarkdown-file-in-r-package
# /pkg/inst/rmd/
# system.file("rmd/file.Rmd", package="packagename")
#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#https://hilaryparker.com/2014/04/29/writing-an-r-package-from-scratch/
# Create Package
# create(myLibrary)

Test Stats only

# Parameters
Selection.Operation <- c("GetGageData","QCRaw", "Aggregate", "SummaryStats")
Selection.Type      <- c("Air","Water","AW","Gage","AWG","AG","WG")
Selection.SUB <- c("Data1_RAW","Data2_QC","Data3_Aggregated","Data4_Stats")
myDir.BASE <- getwd()
#
# Summary Stats, File
## Have to use "file" version.  
## Base version builds file name and expects "DATA" prefix

# Gage File + Other File
myData.Operation <- "SummaryStats" #Selection.Operation[4]
myFile <- c("Data4Stats_01187300_Gage_20130101_20141231.csv"
            ,"Data4Stats_test2_Aw_20130101_20141231.csv")
myDir.import <- file.path(".","data-raw")
myDir.export <- file.path(".","Data4_Stats")

#Leave off myReport.format and get default (docx).
ContDataQC(myData.Operation
           , fun.myDir.import=myDir.import
           , fun.myDir.export=myDir.export
           , fun.myFile=myFile)

SiteID, Date, Time, Date.Time, Parameters, Flag.Parameters

Only one flag per parameter. Do not need individual test flags.

"F" (fail) flags are changed to NA so they are dropped from the statistics that calculated.

Mocked up 2 files.

leppott/ContDataQC documentation built on Jan. 5, 2025, 10:12 a.m.