sandbox/loadDataForTesting.R

In SWS-Methodology/faoswsStandardization: Package to perform standardization of SUA (supply utilization account) data to FBS level

library(data.table)
library(faosws)

GetTestEnvironment(
    baseUrl = "https://hqlprswsas1.hq.un.fao.org:8181/sws",
    token = "c1a2cc38-a9e9-4977-ae6e-4eed6a9568e7"
    ## baseUrl = "https://hqlqasws1.hq.un.fao.org:8181/sws",
    ## token = "90bb0f92-e345-4401-945d-1e43af801167"
)

for(file in dir(path = "~/Documents/Github/faoswsStandardization/R/",
                full.names = TRUE))
    source(file)

extractionRateData = fread("~/Documents/Github/faoswsStandardization/data/extractionRate2011.csv")
shareData = fread("~/Documents/Github/faoswsStandardization/data/shares2011.csv")

setnames(extractionRateData,
         old = c("measuredItemFS", "timePointYears",
                 "Value", "Status"),
         new = c("measuredItemChildFS", "timePointYearsSP",
                 "extractionRate", "flagExtractionRate"))
setnames(shareData, old = "Value", new = "share")
commodityTree = merge(shareData, extractionRateData,
                      by = c("geographicAreaFS", "measuredItemChildFS",
                             "timePointYearsSP"))
commodityTree[, c("Geographic Area.x", "Year.x", "Year.y") := NULL, with = FALSE]
setnames(commodityTree, "Geographic Area.y", "geographicAreaName")
setnames(commodityTree, "Item Parent", "itemParentName")
setnames(commodityTree, "Item Child", "itemChildName")
setnames(commodityTree, "Aupus Required", "aupusRequired")
save(commodityTree, file = "~/Documents/Github/faoswsStandardization/data/commodityTree2011.RData")
load("~/Documents/Github/faoswsStandardization/data/commodityTree2011.RData")
oldCommodityTree = copy(commodityTree)
commodityMap = fread("~/Documents/Github/faoswsStandardization/Old_code_and_documentation/Map_HS2FCL/HS2FCL_Valentina.csv",
                     colClasses = rep("character", 7))
setnames(commodityMap, old = colnames(commodityMap),
         new = c("itemCodeHS2007", "itemNameHS2007", "itemCodeHS2012",
                 "itemNameHS2012", "itemCodeFCL", "itemNameFCL", "bad"))
commodityMap[, bad := NULL]
commodityMap = commodityMap[, list(itemCodeHS2012, itemCodeFCL)]
newColname = "itemCodeHS2012"
oldColname = "itemCodeFCL"

## Plot Commodity Trees in FCL
plotCommodityTrees(
    commodityTree = commodityTree[geographicAreaFS == 1 &
                                  timePointYearsSP == 2011, ],
    parentColname = "measuredItemParentFS",
    childColname = "measuredItemChildFS", extractionColname = "extractionRate",
    dir = "~/Documents/Github/faoswsStandardization/sandbox/", prefix = "",
    adjExtractionRates = TRUE)

plotCommodityTrees(
    commodityTree = commodityTree[, mean(extractionRate),
                                   by = c("measuredItemParentFS",
                                          "measuredItemChildFS")],
    parentColname = "measuredItemParentFS",
    childColname = "measuredItemChildFS", extractionColname = "V1",
    dir = "~/Documents/Github/faoswsStandardization/sandbox/", prefix = "full",
    adjExtractionRates = TRUE)


## Check that everything works:
load("/home/josh/Documents/Github/Working/FBS Example/tradeData.RData")
tradeData = tradeData[, sum(Value), by = c("reportingCountryM49",
                                           "measuredElementTrade",
                                           "measuredItemHS", "timePointYears")]
oldCommodityTree[, measuredItemParentFS :=
                      formatC(as.numeric(measuredItemParentFS), width = 4,
                              flag = "0", format = "d")]
oldCommodityTree[, measuredItemChildFS :=
                      formatC(as.numeric(measuredItemChildFS), width = 4,
                              flag = "0", format = "d")]
newCommodityTree = mapCommodityTree(oldCommodityTree = oldCommodityTree,
                                    commodityMap = commodityMap,
                                    oldColname = "itemCodeFCL",
                                    newColname = "itemCodeHS2012")
tradeData[, measuredItemHS := formatC(as.numeric(measuredItemHS), width = 6,
                                      flag = "0", format = "d")]
## Filter trade data to keep it simple (just look at cereals):
tradeData = tradeData[grepl("^100", measuredItemHS), ]
smallCommodityTree = newCommodityTree[grepl("^100", childItemCodeHS2012) |
                                      grepl("^100", parentItemCodeHS2012), ]
tradeData[, calorieRate := 100]
setnames(tradeData, old = "V1", new = "Value")

levels = getCommodityLevel(smallCommodityTree, "parentItemCodeHS2012",
                           "childItemCodeHS2012", "extractionRate", "share")

newCommodityTree[, byProductFlag := FALSE]

standardizedTrade = tradeData[,
    standardize(nodeData = .SD,
                idColname = "measuredItemHS", quantityColname = "Value",
                calorieRateColname = "calorieRate",
                commodityTree = copy(newCommodityTree),
                parentColname = "parentItemCodeHS2012",
                childColname = "childItemCodeHS2012",
                extractionColname = "extractionRate",
                shareColname = "share",
                targetNodes = levels[level == 0, node],
                byProductColname = "byProductFlag"),
    by = c("reportingCountryM49", "timePointYears", "measuredElementTrade")]
tradeData
standardizedTrade