R/oscn_scrape_epay.R

In openjusticeok/ojo: Access and analyze data from the Open Justice Oklahoma database

oscn_scrape_epay <- function(courts = "TULSA", casetypes, years, case_seqs, updatedb = TRUE, update_freq = 20) {
  if (updatedb == TRUE) {oscn_reset()}
  options(warn=-1)

  for (l in courts) {
    court_tmp <- str_to_upper(l)

    for (m in casetypes) {
      casetype_tmp <- str_to_upper(m)

      for (j in years) {
        caseyear_tmp <- j

        for (k in case_seqs) {

          start <- Sys.time()
          caseseq_tmp <- k
          casenum_tmp <- paste(casetype_tmp, caseyear_tmp,
                               str_pad(caseseq_tmp, side = "left", width = 5, pad = 0),
                               sep = "-")

          ht <- read_html(paste0("https://pay.oscn.net/epayments/find/case?onlinePaymentLocationId=3&caseNumber=", casenum_tmp))

          d <- ht %>%
            html_nodes("div div") %>%
            html_text()

          debt <- str_extract(d, "(?<=\\$)(\\d|,|\\.)*")

          debt <- debt[!is.na(debt)][1] %>%
            str_remove_all(",") %>%
            as.numeric

          debt_tmp <- tibble(oscn_debt_id = paste0(courtlist[[str_to_upper(court_tmp)]], casenum_tmp, "-", Sys.Date()),
                             court = str_to_upper(court_tmp),
                             casenum = casenum_tmp,
                             casetype = casetype_tmp,
                             file_year = caseyear_tmp,
                             debt_amt = debt,
                             debt_date = Sys.Date())

          if (exists("debt_tbl")) {debt_tbl <<- bind_rows(debt_tbl, debt_tmp)
          } else { debt_tbl <<- debt_tmp }

          print(paste(court_tmp, casenum_tmp, "scraped in", Sys.time() - start, "seconds."))

          if (updatedb == TRUE & nrow(debt_tbl) >= update_freq) {
            connect_ojo()
            dbWriteTable(ojo_db, "oscn_debt", debt_tbl, row.names = FALSE, append = TRUE)
            disconnect_ojo()
            debt_tbl <<- tibble()
            print("Database updated.")
          }
        }
      }
    }
    if (updatedb == TRUE) {
      connect_ojo()
      dbWriteTable(ojo_db, "oscn_debt", debt_tbl, row.names = FALSE, append = TRUE)
      disconnect_ojo()
      print("Database updated.")
    }
  }
}