data-raw/data-utils.R

In UchidaMizuki/japancity:

library(tidyverse)
library(fs)
library(reticulate)
library(lubridate)

pkgload::load_all()

# data-utils --------------------------------------------------------------

path_city <- "data-raw/data-city"
path_city_raw <- str_c(path_city, "raw",
                       sep = "/")
path_city_raw_start <- str_c(path_city_raw, "start",
                             sep = "/")
path_city_raw_end <- str_c(path_city_raw, "end",
                           sep = "/")

path_merger <- "data-raw/data-merger"
path_merger_raw <- str_c(path_merger, "raw",
                         sep = "/")

# fs ----------------------------------------------------------------------

dir_clean <- function(path) {
  dir_create(path)
  dir_ls(path) |>
    file_delete()
}

# selenium ----------------------------------------------------------------

ActionChains <- import("selenium.webdriver.common.action_chains")$ActionChains
Select <- import("selenium.webdriver.support.select")$Select
By <- import("selenium.webdriver.common.by")$By

selenium_driver <- function(path,
                            headless = TRUE) {
  webdriver <- import("selenium")$webdriver
  Service <- import("selenium.webdriver.chrome.service")$Service
  ChromeDriverManager <- import("webdriver_manager.chrome")$ChromeDriverManager

  options <- webdriver$ChromeOptions()
  if (headless) {
    options$add_argument("--headless")
  }
  prefs <- list(`download.default_directory` = here::here() |>
                  str_c(path,
                        sep = "/") |>
                  str_replace_all("/", r"(\\)"))
  options$add_experimental_option("prefs", prefs)

  webdriver$Chrome(service = Service(ChromeDriverManager()$install()),
                   options = options)
}

click_city_category <- function(driver) {
  city_category_list <- driver$find_element(By$XPATH, '//td[@data-alias="city_category_list"]')
  city_kd <- 2:7

  for (i in city_kd) {
    xpath <- str_glue('*/input[@name="city_kd[{i}]"]')
    checkbox <- city_category_list$find_element(By$XPATH, xpath)

    if (!checkbox$is_selected()) {
      checkbox$click()
    }
  }
}

click_submit_button <- function(driver) {
  driver$find_element(By$XPATH, '//button[@value="search"]')$click()
}

click_download_button <- function(driver) {
  driver$find_element(By$XPATH, '//li[contains(@class,"js-dbview-download-button")]/button')$click()

  button <- driver$find_element(By$XPATH, '//div[contains(@class,"stat-display_selector-modal-ok")]')
  action <- ActionChains(driver)
  action$move_to_element(button)
  action$click(button)
  action$perform()
}

close_driver <- function(driver, path,
                         size = 1L) {
  if (vec_size(dir_ls(path)) < size) {
    abort("")
  } else {
    driver$close()
  }
}

close_driver <- insistently(close_driver,
                            rate = rate_backoff(max_times = Inf),
                            quiet = FALSE)