Eurostat data"
In convergEU: Monitoring Convergence of EU Countries

knitr::opts_chunk$set(echo = FALSE,eval=FALSE)
require(convergEU)

How to download and filter Eurostat data

The heterogeneity in the structure of different indicators requires some cares. A list of covariates for each indicator is sometimes present besides age and gender thus their values must be set to produce a tidy dataset time by countries.

First, raw data may be downloaded using the option rawDump=T:

download_indicator_EUS(
      indicator_code= convergEU_glb()$metaEUStat$selectorUser[1],
      fromTime = 2005,
      toTime = 2015,
      gender= c(NA,"T","F","M")[1],#c("Total","Females","Males")
      countries =  convergEU_glb()$EU28$memberStates$codeMS,
      rawDump=T )

which is not a tidy dataset. Note that unit and isced11 are auxilary valiables specific for this indicator that must be set to obtain a tidy dataset years by countries.

At this purpose, the argument rawDump=F indicates that bulk data are to be filtered and reshaped, as follows:

myEUSdat <- download_indicator_EUS(
      indicator_code= convergEU_glb()$metaEUStat$selectorUser[1],
      fromTime = 2005,
      toTime = 2015,
      gender= c(NA,"T","F","M")[1],#c("Total","Females","Males")
      ageInterv = NA,
      countries =  convergEU_glb()$EU28$memberStates$codeMS,
      rawDump=F,
      uniqueIdentif = 1)
myEUSdat

where convergEU_glb()\$EU28\$memberStates\$codeMS is a vector of strings for the considered countries, convergEU_glb()\$metaEUStat\$selectorUser[1] contains the name of the indicator and where ageInterv may take a value when an age interval may be specified for a given indicator. The result is list with the following components:

\$res contains the selected data as a tidy tibble (dataset);
\$msg\$gender is a component describing results; \$msg\$gender states that gender was not selected thus it was automatically set;
\$msg\$Age states that age class was automatically set;
\$msg\$Further_Conditioning contains further variables to set in order to obtain a tidy dataset; in particular \$msg\$Further_Conditioning\$current states the value of conditioning specific variables taken from the list of possible ones \$msg\$Further_Conditioning\$available_seleTagLs;
\$msg\$Conditioning contains common conditiongn variables, such as ageInterv and gender.

It is therefore possible to call several time the same function and specify the argument uniqueIdentif as an integer among thos in the first column left of \$msg\$Further_Conditioning\$available_seleTagLs to obtain the same indicator under differt scales and contexts. For example the fifth conditioning context is for males in age interval "Y15-64" is:

myEUSdat <- download_indicator_EUS(
      indicator_code= convergEU_glb()$metaEUStat$selectorUser[1],
      fromTime = 2005,
      toTime = 2015,
      gender= "M",
      ageInterv = "Y15-64",
      countries =  convergEU_glb()$EU28$memberStates$codeMS,
      rawDump=F,
      uniqueIdentif = 5)
myEUSdat

Overview of a selected collection of Eurostat variables

In this section an example is provided for every indicator within the selected collection convergEU_glb()\$metaEUStat\$selectorUser. Note that in some cases a Country may have all values missing.

#eleDBvars <-  list()
for(indicaName in convergEU_glb()$metaEUStat$selectorUser){
  cat("---------------------------------------\n")
  cat("\n---------------------------------------\n\nIndicator:\n",indicaName,"\n")
    scaricati <- download_indicator_EUS(
      indicator_code=indicaName, # metaEUStat$selectorUser
      fromTime = 2005,
      toTime = 2015,
      gender= c(NA,"T","F","M")[1],
      ageInterv = NA,
      countries =  convergEU_glb()$EU28$memberStates$codeMS,
      rawDump=F,
      uniqueIdentif = 1)

    #eleDBvars[[indicaName]]<- names(scaricati)
  print(scaricati)  
  }

# raw download
names(convergEU_glb())
names( convergEU_glb()$metaEUStat )
convergEU_glb()$metaEUStat$subSelection
convergEU_glb()$metaEUStat$selectorUser

ttmp2 <- download_indicator_EUS(
  indicator_code="labourcost_i", # metaEUStat$selectorUser
  fromTime = 2005,
  toTime = 2015,
  gender= c(NA,"T","F","M")[1],#c("Total","Females","Males")
  ageInterv = NA,
  countries =  convergEU_glb()$EU28$memberStates$codeMS,
  rawDump=T)

#nomi variabili di classificazione
nomiCriteria<-list() 
for(vaName in convergEU_glb()$metaEUStat$selectorUser){
  cat("\n\n----------------------------------------------------")
  cat(vaName,"\n\n")
  downTB <- download_indicator_EUS(
            indicator_code=vaName, # metaEUStat$selectorUser
            fromTime = 2005,
            toTime = 2015,
            gender= c(NA,"T","F","M")[1],#c("Total","Females","Males")
            ageInterv = NA,
            countries =  convergEU_glb()$EU28$memberStates$codeMS,
            rawDump=T)
  print(head(downTB))
  nomiCriteria[[vaName]] <- names(downTB)
}
length(nomiCriteria)

# estraggo quelli ovvi e noti: sex,geo,age,time,values
nomiCriteriaMin <- lapply(nomiCriteria,function(vx){
  setdiff(vx,c("sex","geo","age","time","values"))
})

## ora il cartesiano degli attributi popolati 

selezionaSubset <- function(vaName,nomiCriteriaMin){
  downTB <- download_indicator_EUS(
            indicator_code=vaName, # metaEUStat$selectorUser
            fromTime = 2005,
            toTime = 2015,
            gender= c(NA,"T","F","M")[1],#c("Total","Females","Males")
            ageInterv = NA,
            countries =  convergEU_glb()$EU28$memberStates$codeMS,
            rawDump=T)
  estraCarte <- nomiCriteriaMin[[vaName]]
  puntiCampionari <- lapply(estraCarte,function(vx){
        as.character(unlist(unique(downTB[,vx])))
  })
  cartesiano <- expand.grid(puntiCampionari)
  names(cartesiano)<-estraCarte
  cartesianoNum<- mutate(cartesiano,conta=NA)
  for(righeDB in 1:nrow(cartesiano)){
    contatore<- rep(TRUE,nrow(downTB))
    for(aux in estraCarte){
      contatore <- contatore & 
        (unlist(downTB[,aux]) ==  as.character(cartesiano[righeDB,aux])) 
    }
    cartesianoNum$conta[righeDB]<- sum(contatore)
  }
  # select if greater than zero
  estrattore <- which(cartesianoNum$conta >0)
  cartesianoNum<- cartesianoNum[estrattore,]
  list(paste("\n------------------\nSummary for variable:  ",
     vaName,"\n",
     "Please pick one of the following rows:\n",sep=""),
  cartesianoNum) 
}
## testa
# debug(selezionaSubset)
# resFF <- selezionaSubset("lfsa_argaed",nomiCriteriaMin)
# resFF <- selezionaSubset(convergEU_glb()$metaEUStat$selectorUser[20],nomiCriteriaMin)
# cat(resFF [[1]])
# resFF[[2]]

resAll<- list()
punta<- 19
for(indicaName in convergEU_glb()$metaEUStat$selectorUser){
  punta<- punta +1
  print(punta)
   resAll[[indicaName]] <- selezionaSubset(indicaName,nomiCriteriaMin)
   #cat(resAll [[1]])
   #print(resAll[[2]] )
  }

for(indicaName in convergEU_glb()$metaEUStat$selectorUser){
    cat(resAll [[1]])
    print(resAll[[2]] )
  }

# solo elem3enti singoli
selezionaSubsetSingoli <- function(vaName,nomiCriteriaMin){
  downTB <- download_indicator_EUS(
            indicator_code=vaName, # metaEUStat$selectorUser
            fromTime = 2005,
            toTime = 2015,
            gender= c(NA,"T","F","M")[1],#c("Total","Females","Males")
            ageInterv = NA,
            countries =  convergEU_glb()$EU28$memberStates$codeMS,
            rawDump=T)
  estraCarte <- nomiCriteriaMin[[vaName]]
  puntiCampionari <- lapply(estraCarte,function(vx){
        as.character(unlist(unique(downTB[,vx])))
  })
  names(puntiCampionari) <-  estraCarte

  # select if greater than zero
  list(paste("\n------------------\nSummary for variable:  ",
     vaName,"\n",
     "Please pick one of the following items for each list component (variable):\n",sep=""),
  puntiCampionari) 
}



bigSTRres<- " "
for(indicaName in convergEU_glb()$metaEUStat$selectorUser){
  resAll <-  selezionaSubsetSingoli(indicaName,nomiCriteriaMin) 
  if(length(resAll[[2]])>0){
    bigSTRres1<- paste(resAll[[1]], "\n")
    bigSTRres <- paste(bigSTRres,bigSTRres1) 
    for(aux in 1:length(resAll[[2]])){
      bigSTRres2<- paste("\n",names(resAll[[2]])[aux],":\n  ",
      paste(resAll[[2]][[aux]],collapse="; "),"\n\n")
      bigSTRres <- paste(bigSTRres,bigSTRres2) 
     }
    }
  }

cat(bigSTRres,file="/media/fred/STORE/PRJ/2018-TENDER-EU/STEP-1/bitbucketed/convergEU/inst/EUstat_download/toRevise.txt")