R/pubmed.R

In cheungngo/webscr8per: Webscraping tools for academic researches and others

#' Converting keywords to the link of the search result page in pubmed
#'
#' @param words : pubmed search terms
#' @return the link of the search result page in pubmed (size == 200)
#' @examples
#' words2link_pubmed("(PTSD OR Depression) AND Ketamine")
words2link_pubmed = function(words) {
  code = gsub(" ", "%20", words)
  link = paste("https://pubmed.ncbi.nlm.nih.gov/?term=",
               code,
               "&size=200&page=",
               sep = "")
  return(link)
}

#' Extracting the searchpage in pubmed
#'
#' @param path : path generated by the function words2link_pubmed()
#' @param no_of_results : total results appeared in pubmed
#' @return two vectors of variables: titles and hyperlinks
#' @examples
#' library(zeallot)
#' c(title, link) %<-% extract_searchpage_pubmed(link, 10) # the first 10 entries
extract_searchpage_pubmed = function(path, no_of_results) {

  library(rvest)

  xpath1 = '//*[@id="search-results"]/section/div['
  xpath2 = ']/div/article['
  xpath3 = ']/div[2]/div[1]/a'
  per_page = 200
  no_page = (no_of_results %/% per_page) + 1

  titles = c()
  hrefs = c()

  for (i in 1:no_page) {
    pathi = xml2::read_html(paste(path, i, sep = ""))
    for (j  in 1:per_page) {

      if (((i-1)*200 + j) > no_of_results) {next}

      xp = paste(xpath1, i, xpath2, j, xpath3, sep = "")
      title = pathi %>%
        rvest::html_node(xpath = xp) %>%
        rvest::html_text()
      title = stringr::str_squish(title)
      titles = c(titles, title)

      href = pathi %>%
        rvest::html_node(xpath = xp) %>%
        rvest::html_attr("href")
      href = paste("https://pubmed.ncbi.nlm.nih.gov", href, sep = "")
      hrefs = c(hrefs, href)

    }
  }

  return(list(titles, hrefs))
}

#' Extracting the abstract from pubmed
#'
#' @param link : link obtained by the function extract_searchpage_pubmed()
#' @return abstract of the article (a string)
#' @examples
#' c(title, link) %<-% extract_searchpage_pubmed(words2link_pubmed("ptsd ketamine"), 211)
#' abstract = c()
#' for (i in 1:10) {
#'   abstract = c(abstract, extract_abstract_pubmed(link[i]))
#' }
extract_abstract_pubmed = function(link) {
  href = xml2::read_html(link)
  abs = href %>% html_node(xpath = '//*[@id="enc-abstract"]') %>% html_text()
  abs = stringr::str_squish(abs)
  return(abs)
}