Eurostat data"
In convergEU: Monitoring Convergence of EU Countries

knitr::opts_chunk$set(echo = FALSE)
library(convergEU)
library(tidyverse)

How to download and filter Eurostat data

The heterogeneity in the structure of different indicators requires some cares. Below a list of covariates is shown for each indicator and one of them must be selected to produce a tidy dataset time by countries.

# raw download
names(convergEU_glb())
names( convergEU_glb()$metaEUStat )
convergEU_glb()$metaEUStat$subSelection
convergEU_glb()$metaEUStat$selectorUser

ttmp2 <- download_indicator_EUS(
  indicator_code="labourcost_i", # metaEUStat$selectorUser
  fromTime = 2005,
  toTime = 2015,
  gender= c(NA,"T","F","M")[1],#c("Total","Females","Males")
  ageInterv = NA,
  countries =  convergEU_glb()$EU28$memberStates$codeMS,
  rawDump=T)

# ttmp2 <- mutate(ttmp2, etica= paste("unit= ",.data$unit,"; item: ",.data$na_item,sep=""))
# unique(ttmp2$etica)
# 
# filter(ttmp2, unit=="PPS", na_item=="D1_SAL_HW")




#nomi variabili di classificazione
nomiCriteria<-list() 
for(vaName in convergEU_glb()$metaEUStat$selectorUser){
  cat("\n\n----------------------------------------------------")
  cat(vaName,"\n\n")
  downTB <- download_indicator_EUS(
            indicator_code=vaName, # metaEUStat$selectorUser
            fromTime = 2005,
            toTime = 2015,
            gender= c(NA,"T","F","M")[1],#c("Total","Females","Males")
            ageInterv = NA,
            countries =  convergEU_glb()$EU28$memberStates$codeMS,
            rawDump=T)
  print(head(downTB))
  nomiCriteria[[vaName]] <- names(downTB)
}
length(nomiCriteria)

# estraggo quelli ovvi e noti: sex,geo,age,time,values
nomiCriteriaMin <- lapply(nomiCriteria,function(vx){
  setdiff(vx,c("sex","geo","age","time","values"))
})

## ora il cartesiano degli attributi popolati 

selezionaSubset <- function(vaName,nomiCriteriaMin){
  downTB <- download_indicator_EUS(
            indicator_code=vaName, # metaEUStat$selectorUser
            fromTime = 2005,
            toTime = 2015,
            gender= c(NA,"T","F","M")[1],#c("Total","Females","Males")
            ageInterv = NA,
            countries =  convergEU_glb()$EU28$memberStates$codeMS,
            rawDump=T)
  estraCarte <- nomiCriteriaMin[[vaName]]
  puntiCampionari <- lapply(estraCarte,function(vx){
        as.character(unlist(unique(downTB[,vx])))
  })
  cartesiano <- expand.grid(puntiCampionari)
  names(cartesiano)<-estraCarte
  cartesianoNum<- mutate(cartesiano,conta=NA)
  for(righeDB in 1:nrow(cartesiano)){
    contatore<- rep(TRUE,nrow(downTB))
    for(aux in estraCarte){
      contatore <- contatore & 
        (unlist(downTB[,aux]) ==  as.character(cartesiano[righeDB,aux])) 
    }
    cartesianoNum$conta[righeDB]<- sum(contatore)
  }
  # select if greater than zero
  estrattore <- which(cartesianoNum$conta >0)
  cartesianoNum<- cartesianoNum[estrattore,]
  list(paste("\n------------------\nSummary for variable:  ",
     vaName,"\n",
     "Please pick one of the following rows:\n",sep=""),
  cartesianoNum) 
}
## testa
# debug(selezionaSubset)
# resFF <- selezionaSubset("lfsa_argaed",nomiCriteriaMin)
# resFF <- selezionaSubset(convergEU_glb()$metaEUStat$selectorUser[20],nomiCriteriaMin)
# cat(resFF [[1]])
# resFF[[2]]

resAll<- list()
punta<- 19
for(indicaName in convergEU_glb()$metaEUStat$selectorUser){
  punta<- punta +1
  print(punta)
   resAll[[indicaName]] <- selezionaSubset(indicaName,nomiCriteriaMin)
   #cat(resAll [[1]])
   #print(resAll[[2]] )
  }

for(indicaName in convergEU_glb()$metaEUStat$selectorUser){
    cat(resAll [[1]])
    print(resAll[[2]] )
  }

# solo elem3enti singoli
selezionaSubsetSingoli <- function(vaName,nomiCriteriaMin){
  downTB <- download_indicator_EUS(
            indicator_code=vaName, # metaEUStat$selectorUser
            fromTime = 2005,
            toTime = 2015,
            gender= c(NA,"T","F","M")[1],#c("Total","Females","Males")
            ageInterv = NA,
            countries =  convergEU_glb()$EU28$memberStates$codeMS,
            rawDump=T)
  estraCarte <- nomiCriteriaMin[[vaName]]
  puntiCampionari <- lapply(estraCarte,function(vx){
        as.character(unlist(unique(downTB[,vx])))
  })
  names(puntiCampionari) <-  estraCarte

  # select if greater than zero
  list(paste("\n------------------\nSummary for variable:  ",
     vaName,"\n",
     "Please pick one of the following items for each list component (variable):\n",sep=""),
  puntiCampionari) 
}



bigSTRres<- " "
for(indicaName in convergEU_glb()$metaEUStat$selectorUser){
  resAll <-  selezionaSubsetSingoli(indicaName,nomiCriteriaMin) 
  if(length(resAll[[2]])>0){
    bigSTRres1<- paste(resAll[[1]], "\n")
    bigSTRres <- paste(bigSTRres,bigSTRres1) 
    for(aux in 1:length(resAll[[2]])){
      bigSTRres2<- paste("\n",names(resAll[[2]])[aux],":\n  ",
      paste(resAll[[2]][[aux]],collapse="; "),"\n\n")
      bigSTRres <- paste(bigSTRres,bigSTRres2) 
     }
    }
  }

cat(bigSTRres,file="/media/fred/STORE/PRJ/2018-TENDER-EU/STEP-1/bitbucketed/convergEU/inst/EUstat_download/toRevise.txt")