NOTES

Setup

library(rlang,
        include.only = "%|%")
library(magrittr,
        include.only = c("%>%", "%<>%", "%T>%", "%!>%", "%$%"))

Define data

all_ballot_dates

FOKUS-covered ballot dates

all_ballot_dates <-
  fs::dir_ls(path = "data-raw/questionnaire",
             type = "file",
             regexp = "/\\d{4}-\\d{2}-\\d{2}\\.toml$") %>%
  stringr::str_extract(pattern = "\\d{4}-\\d{2}-\\d{2}(?=\\.toml$)") %>%
  lubridate::as_date()

raw_qstnr

Raw questionnaire data as a structured list

raw_qstnr <- pal::toml_read("data-raw/questionnaire/questionnaire.toml")

raw_qstnrs_suppl

Raw supplemental date-specific questionnaire data as a structured list

raw_qstnrs_suppl <-
  all_ballot_dates %>%
  magrittr::set_names(., .) %>%
  purrr::map(~ pal::toml_read(glue::glue("data-raw/questionnaire/{.x}.toml")))

cantons_at

FOKUS-covered cantons per ballot date

cantons_at <-
  raw_qstnrs_suppl %>%
  purrr::map(~ .x %>%
               purrr::chuck("mode") %>%
               names())

all_cantons

all_cantons <-
  cantons_at %>%
  purrr::list_c(ptype = character()) %>%
  unique()

all_ballot_types

Possible ballot types

all_ballot_types <- c("referendum",
                      "election")

all_lvls

Possible political levels

all_lvls <- c("cantonal",
              "federal")

all_prcds

Possible election procedures

all_prcds <- c("proportional",
               "majoritarian")

all_proposal_types

NOTES:

all_proposal_types <- c("citizens' initiative",
                        "direct counterproposal",
                        "indirect counterproposal",
                        "mandatory referendum",
                        "optional referendum")

all_qstn_groups

Possible standardized question groups

all_qstn_groups <- c("argument", "main_motive", "skill_question")

all_argument_sides

Possible proposal argument sides

all_argument_sides <- c("pro",
                        "contra")

all_main_motive_types

Possible main motive types

all_main_motive_types <- c("yes",
                           "no")

all_election_seat_types

Possible election seat types

all_election_seat_types <- c("vacant",
                             "total")

all_response_option_types

Response option types defined in the raw FOKUS questionnaire data

all_response_option_types <- names(raw_qstnr$response_options)

all_postal_dispatch_types

Postal dispatch types used in the raw supplemental date-specific questionnaire data.

all_postal_dispatch_types <-
  cantons_at %>%
  purrr::imap(function(cantons, ballot_date) {

    cantons %>% purrr::map(~ names(raw_qstnrs_suppl[[ballot_date]]$mode[[.x]]$postal_dispatch$type))
  }) %>%
  unlist(use.names = FALSE) %>%
  unique()

all_postal_dispatch_ways

Postal dispatch ways used in the raw supplemental date-specific questionnaire data.

all_postal_dispatch_ways <-
  cantons_at %>%
  purrr::imap(function(cantons, ballot_date) {

    cantons %>% purrr::map(~ raw_qstnrs_suppl[[ballot_date]]$mode[[.x]]$postal_dispatch$type)
  }) %>%
  unlist(use.names = FALSE) %>%
  unique() %>%
  sort()

all_langs

Possible languages (for proposal names etc.).

all_langs <- c("de",
               "en")

all_name_types

Possible name types for various entities.

all_name_types <- c("short",
                    "long")

col_types

col_types <- list(ballot_date            = readr::col_date(),
                  ballot_votes_received  = readr::col_integer(),
                  electorate_total       = readr::col_integer(),
                  entity                 = readr::col_character(),
                  evoting_votes_received = readr::col_integer(),
                  fso_id                 = readr::col_integer(),
                  isolated_votes_empty   = readr::col_integer(),
                  isolated_votes_invalid = readr::col_integer(),
                  postal_votes_invalid   = readr::col_integer(),
                  postal_votes_received  = readr::col_integer(),
                  slots_empty            = readr::col_integer(),
                  tickets_modified       = readr::col_integer(),
                  tickets_partyless      = readr::col_integer(),
                  tickets_unmodified     = readr::col_integer(),
                  tickets_valid          = readr::col_integer(),
                  turnout                = readr::col_double(),
                  votes_empty            = readr::col_integer(),
                  votes_invalid          = readr::col_integer(),
                  votes_no               = readr::col_integer(),
                  votes_received         = readr::col_integer(),
                  votes_total            = readr::col_integer(),
                  votes_valid            = readr::col_integer(),
                  votes_yes              = readr::col_integer(),
                  voters_total           = readr::col_double())

dicts

dicts <- list(colnames = list(ballots = c(BFSNR                     = "fso_id",
                                          DATUM                     = "ballot_date",
                                          Bezirk                    = "entity",
                                          `Bezirk/Liste`            = "entity",
                                          NAME                      = "entity",
                                          `Anzahl Stimmberechtigte` = "electorate_total",
                                          STIMMBERECHTIGTE          = "electorate_total",
                                          WAHLBER                   = "electorate_total",
                                          WAHLBERECHTIGTE           = "electorate_total",
                                          EINGELANGTE               = "votes_received",
                                          LEERE                     = "votes_empty",
                                          UNGUELTIGE                = "votes_invalid",
                                          GUELTIGE                  = "votes_valid",
                                          JA_STIMMEN                = "votes_yes",
                                          NEIN_STIMMEN              = "votes_no",
                                          TOTAL                     = "votes_total",
                                          Total                     = "votes_total",
                                          `Total Wählerzahlen`      = "voters_total",
                                          WAEHLENDE                 = "voters_total",
                                          GUELT_WAHLZ               = "tickets_valid",
                                          UNVERAE_WAHLZ             = "tickets_unmodified",
                                          VERAE_WAHLZ               = "tickets_modified",
                                          WAHLZ_O_PARTEI            = "tickets_partyless",
                                          LEERE_LINIEN              = "slots_empty",
                                          WAHLBET                   = "turnout",
                                          Wahlbeteiligung           = "turnout",
                                          EINGELBRIEF               = "postal_votes_received",
                                          BRIEF_EINGEL              = "postal_votes_received",
                                          BRIEFEINGEL               = "postal_votes_received",
                                          UNGUELTIGBRIEF            = "postal_votes_invalid",
                                          BRIEF_UNGUELTIG           = "postal_votes_invalid",
                                          BRIEFUNGUELTG             = "postal_votes_invalid",
                                          EINGELURNEN               = "ballot_votes_received",
                                          URNEN_EINGEL              = "ballot_votes_received",
                                          URNEN_STIMMABG            = "ballot_votes_received",
                                          EVOTING_EINGEL            = "evoting_votes_received",
                                          VEREINZELTLEERE           = "isolated_votes_empty",
                                          VEREINZELTUNGUE           = "isolated_votes_invalid",
                                          VEREINZELTE               = fokus::response_options(type = "other",
                                                                                              lang = "de",
                                                                                              subtypes = "candidate"))))

fokus_private_structure

Structure of the fokus_private repository.

fokus_private_structure <- list("fokus_private/" = list("generated/" = list("for-polling-agency/" = list("{ballot_date}_{canton}_easyvote_municipalities.csv",
                                                                                                         "{ballot_date}_{canton}_print_recipients.csv",
                                                                                                         "{ballot_date}_{canton}_qr_codes.zip"),
                                                                            "survey_data_de_{ballot_date}_{canton}.rds",
                                                                            "survey_data_en_{ballot_date}_{canton}.rds",
                                                                            "survey_data_merged_de_{ballot_date}_{canton}.rds",
                                                                            "survey_data_merged_en_{ballot_date}_{canton}.rds"),
                                                        "raw/" = list("easyvote_municipalities_{ballot_date}_{canton}.csv",
                                                                      "online_participation_codes_{ballot_date}_{canton}.txt",
                                                                      "survey_data_{ballot_date}_{canton}.xlsx",
                                                                      "survey_data_{ballot_date}_{canton}_*.xlsx",
                                                                      "survey_data_preliminary_{ballot_date}_{canton}.xlsx",
                                                                      "voting_register_data_extra_{date_delivery_statistical_office}_{canton}.xlsx",
                                                                      "voting_register_ids_{ballot_date}_{canton}.csv",
                                                                      "..."),
                                                        "..."))

md_snippets

md_snippets <-
  fs::dir_ls(path = "data-raw/snippets/",
             type = "file",
             glob = "*.md") %>%
  magrittr::set_names(value = fs::path_ext_remove(fs::path_file(.))) %>%
  purrr::map(brio::read_file)

phrased_terms

phrased_terms <- tibble::tribble(
  ~term,                  ~value,                   ~en,                      ~de,
  "ballot_type",          "referendum",             "referendum",             "Abstimmung",
  "ballot_type",          "election",               "election",               "Wahl",
  "lvl",                  "municipal",              "municipal",              "kommunal",
  "lvl",                  "cantonal",               "cantonal",               "kantonal",
  "lvl",                  "federal",                "federal",                "eidgenössisch",
  "main_motive_type",     "yes",                    "yes",                    "Ja",
  "main_motive_type",     "no",                     "no",                     "Nein",
  "postal_dispatch_type", "invitation",             "invitation letter",      "Einladungsschreiben",
  "postal_dispatch_type", "reminder",               "reminder letter",        "Erinnerungsschreiben",
  "postal_dispatch_type", "prepaid_reply_envelope", "prepaid reply envelope", "vorfrankiertes Rückantwortcouvert",
  "postal_dispatch_way",  "a",                      "A priority mail",        "A-Post",
  "postal_dispatch_way",  "b",                      "B mail",                 "B-Post",
  "postal_dispatch_way",  "b_bulk",                 "B bulk mailing",         "B-Post-Massensendung",
  "side",                 "pro",                    "pro",                    "Pro",
  "side",                 "contra",                 "contra",                 "Kontra"
)

phrased_terms_tidy

Long-data version of phrased_terms for internal use.

phrased_terms_tidy <- tidyr::pivot_longer(data = phrased_terms,
                                          cols = all_of(all_langs),
                                          names_to = "lang",
                                          values_to = "phrase")

pkg_config

pkg_config <-
  tibble::tibble(key = character(),
                 default_value = list(),
                 default_value_dynamic = character(),
                 description = character()) |>
  tibble::add_row(key = "ballot_date",
                  default_value = list(all_ballot_dates[1L]),
                  description = paste0("FOKUS-covered ballot date to fall back to in various functions of this package when it isn't explicitly specified. ",
                                       "Basically a means to globally define the ballot date to be processed. One of ",
                                       cli::ansi_collapse(pal::as_md_vals(as.character(all_ballot_dates)),
                                                          last = " or "),
                                       ".")) |>
  tibble::add_row(key = "canton",
                  default_value = list(all_cantons[1L]),
                  description = paste0("FOKUS-covered canton name to fall back to in various functions of this package when it isn't explicitly specified. ",
                                       "Basically a means to globally define the canton to be processed. One of ",
                                       cli::ansi_collapse(pal::as_md_vals(as.character(all_cantons)),
                                                          last = " or "),
                                       ".")) |>
  tibble::add_row(key = "lang",
                  default_value = list(all_langs[1L]),
                  description = paste0("Language to fall back to in various functions of this package when it isn't explicitly specified. Basically a means to",
                                       " globally define the language to process. One of ",
                                       cli::ansi_collapse(pal::as_md_vals(as.character(all_langs)),
                                                          last = " or "),
                                       ".")) |>
  tibble::add_row(key = "global_max_cache_age",
                  default_value = list("1 day"),
                  description = pkgsnip::md_snip("opt_global_max_cache_age")) |>
  tibble::add_row(key = "token_repo_private",
                  default_value_dynamic = 'Sys.getenv("GITLAB_COM_TOKEN")',
                  description = glue::glue("[Personal access token of a gitlab.com account](https://gitlab.com/-/user_settings/personal_access_tokens) with ",
                                           "access to the [private FOKUS repository]({fokus:::url_repo_private()})."))

qstnr_item_keys

Questionnaire item keys supported in the raw FOKUS questionnaire data

qstnr_item_keys <- tibble::tribble(

  ~key,                         ~is_scalar, ~is_iterator, ~is_excluder, ~is_optional, ~type,       ~default,      ~remarks,
  "lvl",                        FALSE,      TRUE,         FALSE,        FALSE,        "character", '"?"',         "political-level loop iterator",
  "i",                          FALSE,      TRUE,         FALSE,        FALSE,        "integer",   'NA_integer_', "2nd-level loop iterator",
  "j",                          FALSE,      TRUE,         FALSE,        FALSE,        "integer",   'NA_integer_', paste0("3rd-level loop iterator, i.e. ",
                                                                                                                           "\"for each `lvl`, iterate over ",
                                                                                                                           "each `i`, and in turn iterate over",
                                                                                                                           " every `j`\""),
  "block",                      TRUE,       FALSE,        FALSE,        FALSE,        "character", '*top-level block name*', paste0("holds the name of the ",
                                                                                                                                  "respective top-level block;",
                                                                                                                                  " filled automatically ",
                                                                                                                                  "during parsing"),
  "variable_name",              TRUE,       FALSE,        FALSE,        FALSE,        "character", '"???"',       paste0("aka \"Variablenname\"; mustn't have ",
                                                                                                                         "any subkeys (see below); usually not",
                                                                                                                         " sensible to be inherited since it ",
                                                                                                                         "has to be unique"),
  "who",                        TRUE,       FALSE,        FALSE,        FALSE,        "character", '"alle"',      "aka \"Wer\"",
  "topic",                      TRUE,       FALSE,        FALSE,        TRUE,         "character", NA_character_, "aka \"Thema\"",
  "question_intro_i",           TRUE,       FALSE,        FALSE,        TRUE,         "character", NA_character_, paste0("1st-priority part of \"Frage\", only",
                                                                                                                         " included if `i == 1` and ",
                                                                                                                         "`j %in% c(1, NA)`"),
  "question_intro_j",           TRUE,       FALSE,        FALSE,        TRUE,         "character", NA_character_, paste0("2nd-priority part of \"Frage\", only",
                                                                                                                         " included if `j == 1`"),
  "question",                   TRUE,       FALSE,        FALSE,        TRUE,         "character", NA_character_, paste0("last-priority part of \"Frage\", ",
                                                                                                                         "always included"),
  "question_full",              TRUE,       FALSE,        FALSE,        TRUE,         "character", NA_character_, paste0("alternative fully formulated version",
                                                                                                                         " of \"Frage\" that can refer to ",
                                                                                                                         "`question`"),
  "question_common",            TRUE,       FALSE,        FALSE,        TRUE,         "character", NA_character_, paste0("version of `question` that stays the",
                                                                                                                         " same across ballot dates"),
  "variable_label",             TRUE,       FALSE,        FALSE,        FALSE,        "character", '"???"',       "aka \"Variablenlabel\"",
  "variable_label_common",      TRUE,       FALSE,        FALSE,        TRUE,         "character", NA_character_, paste0("version of `variable_label` that ",
                                                                                                                         "stays the same across ballot dates"),
  "response_options",           FALSE,      FALSE,        FALSE,        TRUE,         "character", NA_character_, "aka \"Antwortoptionen\"",
  "variable_values",            FALSE,      FALSE,        FALSE,        TRUE,         "integer",   NA_character_, "aka \"Variablenausprägungen\"",
  "value_labels",               FALSE,      FALSE,        FALSE,        TRUE,         "character", NA_character_, "aka \"Ausprägungslabels\"",
  "value_scale",                TRUE,       FALSE,        FALSE,        FALSE,        "character", '"nominal"',   paste0(
    "item's [scale of measure (aka \"level of measurement\")](https://en.wikipedia.org/wiki/Level_of_measurement); possible values include ",
    "`\"binary\"`, `\"nominal\"`, `\"ordinal_ascending\"`, `\"ordinal_descending\"`, `\"interval\"` and `\"ratio\"`"),
  "allow_multiple_answers",     TRUE,       FALSE,        FALSE,        FALSE,        "logical",   'FALSE',       "aka \"Mehrfachnennungen\"",
  "randomize_response_options", TRUE,       FALSE,        FALSE,        FALSE,        "logical",   'FALSE',       paste0("whether or not the ",
                                                                                                                         "`response_options` are displayed in ",
                                                                                                                         "randomized order to online ",
                                                                                                                         "respondents (`response_options`) ",
                                                                                                                         "with code `80`, `90` and `99` are ",
                                                                                                                         "still excluded from randomization)"),
  "is_mandatory",               TRUE,       FALSE,        FALSE,        FALSE,        "logical",   'FALSE',       "aka \"Antwort obligatorisch\"",
  "ballot_types",               FALSE,      FALSE,        TRUE,         FALSE,        "character", 'c("referendum", "election")',
  "ballot-type-specific in- and exclusion of the respective item(s)",
  "include",                    TRUE,       FALSE,        TRUE,         FALSE,        "logical",   'TRUE',        paste0("ballot-date-wide in- and exclusion ",
                                                                                                                         "of the respective item(s)")
) %>%
  # add list-column with actual default values
  dplyr::mutate(default_val = purrr::map2(.x = default,
                                          .y = type,
                                          .f = ~ {
                                            if (is.na(.x) || stringr::str_detect(.x, "^\\*.*\\*$")) {
                                              .x
                                            } else {
                                              eval(parse(text = glue::glue("as.{.y}({.x})")))
                                            }
                                          }))

qstnr_item_keys_multival

Pure convenience.

qstnr_item_keys_multival <-
  qstnr_item_keys %>%
  dplyr::filter(!is_iterator & !is_scalar & !is_excluder) %$%
  key

!qstnr_item_subkeys

Not yet used anywhere, thus disabled.

qstnr_item_subkeys <- c("true",
                        "false",
                        unique(cantons_at),
                        stringr::str_remove_all(all_ballot_dates, "-"),
                        # TODO: add valid date ranges
                        "referendum",
                        "election",
                        "default")

qstnr_non_item_lvls

qstnr_non_item_lvls <- c("title",
                         "who",
                         "party",
                         "response_options",
                         "footnote",
                         "link")

repo_private_default_branch

repo_private_default_branch <- gitlab::project_default_branch(id_project = fokus:::repo_private_proj_id,
                                                              token = pal::pkg_config_val(key = "token_repo_private",
                                                                                          pkg = "fokus"))

shortening_rules

Variable name shortening rules

The rules will be applied one by one in the order they are listed below by shorten_var_names().

shortening_rules <- tibble::tribble(

  ~string,                                      ~replacement,                          ~allowed_at,

  # sophisticated rules
  "cantonal_government_parliament",             "cgovparl",                            "begin-middle-end",
  "cantonal_government",                        "cgov",                                "begin-middle-end",
  "cantonal_parliament",                        "cparl",                               "begin-middle-end",
  "cantonal_proposals",                         "cps",                                 "begin-middle-end",
  "cantonal_proposal",                          "cp",                                  "begin-middle-end",
  "federal_proposals",                          "fps",                                 "begin-middle-end",
  "federal_proposal",                           "fp",                                  "begin-middle-end",
  "cantonal_proportional_elections",            "cpes",                                "begin-middle-end",
  "cantonal_proportional_election",             "cpe",                                 "begin-middle-end",
  "federal_proportional_elections",             "fpes",                                "begin-middle-end",
  "federal_proportional_election",              "fpe",                                 "begin-middle-end",
  "cantonal_majoritarian_elections",            "cmes",                                "begin-middle-end",
  "cantonal_majoritarian_election",             "cme",                                 "begin-middle-end",
  "federal_majoritarian_elections",             "fmes",                                "begin-middle-end",
  "federal_majoritarian_election",              "fme",                                 "begin-middle-end",
  "cantonal_elections",                         "ces",                                 "begin-middle-end",
  "cantonal_election",                          "ce",                                  "begin-middle-end",
  "federal_elections",                          "fes",                                 "begin-middle-end",
  "federal_election",                           "fe",                                  "begin-middle-end",
  "cantonal",                                   "c",                                   "begin-middle-end",
  "federal",                                    "f",                                   "begin-middle-end",
  "time",                                       "t",                                   "begin",
  "other",                                      "o",                                   "end",
  "reduced",                                    "rdc",                                 "end",

  # simple rules
  "applications",                               "apps",                                "begin-middle-end",
  "agglomeration",                              "agglo",                               "begin-middle-end",
  "attitude",                                   "att",                                 "begin-middle-end",
  "booklet",                                    "bkl",                                 "begin-middle-end",
  "candidate",                                  "cand",                                "begin-middle-end",
  "competent",                                  "cmp",                                 "begin-middle-end",
  "comprehension",                              "cmprh",                               "begin-middle-end",
  "contentment",                                "cntmnt",                              "begin-middle-end",
  "convincing",                                 "cnv",                                 "begin-middle-end",
  "custom",                                     "cm",                                  "begin-middle-end",
  "decision",                                   "dcsn",                                "begin-middle-end",
  "groups",                                     "grps",                                "begin-middle-end",
  "hypothetical",                               "hypo",                                "begin-middle-end",
  "importance",                                 "imp",                                 "begin-middle-end",
  "ineffectiveness",                            "ineff",                               "begin-middle-end",
  "information_source",                         "info_src",                            "begin-middle-end",
  "infrastructure",                             "infra",                               "begin-middle-end",
  "insurance",                                  "insrnc",                              "begin-middle-end",
  "management",                                 "mgmt",                                "begin-middle-end",
  "modification",                               "mod",                                 "begin-middle-end",
  "newspapers",                                 "newsp",                               "begin-middle-end",
  "official",                                   "offi",                                "begin-middle-end",
  "options",                                    "opts",                                "begin-middle-end",
  "political",                                  "pol",                                 "begin-middle-end",
  "positioning",                                "pos",                                 "begin-middle-end",
  "public",                                     "pub",                                 "begin-middle-end",
  "reader_letters_comments",                    "rlc",                                 "begin-middle-end",
  "reduction",                                  "red",                                 "begin-middle-end",
  "spending",                                   "spend",                               "begin-middle-end",
  "switzerland",                                "ch",                                  "begin-middle-end",
  "typology",                                   "typ",                                 "begin-middle-end",

  # lazy rules
  ## ubiqiutous
  "environmentalism_vs_economic_prosperity",    "env_vs_econ",                         "begin-middle-end",
  "equal_opportunity_foreigners",               "equal_opportunity_foreign",           "begin-middle-end",
  "equivalised_income",                         "equi_inc",                            "begin-middle-end",
  "reason_non_participation",                   "reason_non_part",                     "begin-middle-end",
  "weight_participation",                       "weight_prt",                          "begin-middle-end",
  "welfare_state_vs_self_responsibility",       "welfare_vs_self_respon",              "begin-middle-end",
  ## rare
  "demokratiefabrik_argumentary",               "demfab_argmty",                       "begin-middle-end",
  "demokratiefabrik",                           "demfab",                              "begin-middle-end",
  "demoscan_flyer",                             "demsc_fly",                           "begin-middle-end",
  "demoscan",                                   "demsc",                               "begin-middle-end",
  "infectious_disease_specialists",             "infct_disease_spclsts",               "begin-middle-end"
)

Write data

Save all the small data objects as a single internal file R/sysdata.rda. Note that when documenting them, they must be explicitly @exported to be available to package users.

usethis::use_data(all_ballot_dates,
                  raw_qstnr,
                  raw_qstnrs_suppl,
                  cantons_at,
                  all_cantons,
                  all_ballot_types,
                  all_lvls,
                  all_prcds,
                  all_proposal_types,
                  all_qstn_groups,
                  all_argument_sides,
                  all_main_motive_types,
                  all_election_seat_types,
                  all_response_option_types,
                  all_postal_dispatch_types,
                  all_postal_dispatch_ways,
                  all_langs,
                  all_name_types,
                  col_types,
                  dicts,
                  fokus_private_structure,
                  md_snippets,
                  phrased_terms,
                  phrased_terms_tidy,
                  pkg_config,
                  qstnr_item_keys,
                  qstnr_item_keys_multival,
                  qstnr_non_item_lvls,
                  repo_private_default_branch,
                  shortening_rules,
                  internal = TRUE,
                  overwrite = TRUE,
                  compress = "xz",
                  version = 3L)


zdaarau/fokus documentation built on Dec. 24, 2024, 10:47 p.m.