R/hd_catalog_scrape.R

In burtonlueth/webPrices: Screen scraping for collecting wholesale product pricing on web

#' Get product and price details for all products in a section on Home Depot from main url
#' @param links A list of character strings for web urls
#' @return Returns a single data frame with data for every product within a catalog section
#' @export
#' @examples
#' \dontrun{
#' x <- "http://www.homedepot.com/b/Electrical-Dimmers-Switches-Outlets/N-5yc1vZc34h"
#' hd_catalog_scrape(x)
#' }
hd_catalog_scrape <- function(x) {
  pages <- x %>% hd_build_multi_page_links %>% unlist
  prods <- purrr::map(pages, hd_get_product_links) %>% unlist %>% hd_add_main_url
  hd_tidy_scrape(prods)
  d[grepl("modelno", purrr::map(d, ~names(.x[3])))] %>% purrr::reduce(rbind)
}