R/search.R

In bhakyuz/sahibinden: Crunch Some Data From sahibinden.com

#' @title Get search results from ajax
#' @description Experimental funct to get search results for for 'satilik'
#' from map based data 
#' @param print_steps Print out proress 
#' @return html document of search results, can be written in disk visk xml2::write_xml()
#' @examples 
#' search()
search <- function(print_steps = F){
  # curl examples
  # curl 'https://www.sahibinden.com/ajax/search/map?pagingSize=50&m%3AfacetedAttributeFilters=locationFacet&address_country=1&language=tr&category=16633&pagingOffset=0&price_currency=1' -H 'User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:66.0) Gecko/20100101 Firefox/66.0' -H 'Accept: application/json, text/javascript, */*; q=0.01' -H 'Accept-Language: en-US,en;q=0.5' --compressed -H 'Referer: https://www.sahibinden.com/haritada-emlak-arama/satilik'
  base_url <- "https://www.sahibinden.com/ajax/search/map"
  final_url <- httr::parse_url(base_url)
  final_url$query <- c(
    "pagingSize" = 50,
    "pagingOffset" = 0,
    "m:facetedAttributeFilters" = "locationFacet",
    "category" = 16623,
    "address_country" = 1, 
    "address_town" = sapply(421:422, function(x) list(address_town = x)),
    "address_city" = 3,
    "price_currency" = 1,
    "sorting" = "date_desc",
    "m:geoLocationSearch" = T,
    "searchMeta" = T,
    "m:attributeFacets" = T
    )
  
  response <- httr::GET(
    url = httr::build_url(final_url), 
    httr::add_headers("Accept" = "application/json, text/javascript, */*; q=0.01"),
    httr::add_headers("Accept-Language" = "en-US,en;q=0.5"),
    httr::add_headers("Referer" = "https://www.sahibinden.com/haritada-emlak-arama/satilik"),
    if(print_steps) httr::verbose() else NULL
  )
  
  if(httr::status_code(response) == 200) {
    if(print_steps) cat("Search results are retrieved with success\n")
    content <- httr::content(response)
    classifieds <- lapply(content$classifieds, flatten) %>% 
      dplyr::bind_rows() %>% 
      dplyr::as.tbl()
    
    return(list(raw = content, classifieds = classifieds, paging = content$paging))
  } else {
    if(print_steps) cat("Data not found!\n")
    return(NULL)
  }
}