R/data_scrape.R

In RandalMorris/AnalyticsFootball: Fantasy Data from Various Data Sources

#' Scrape Data, fix data and export
#'
#'
#' @export
#'
data_scrape <- function(){

  #########Scraping Data Sources#########
  {message("Scraping Data Sources")

    my_scrape <<- scrape_data(src= c("CBS", "FantasyPros", "FantasySharks", "FFToday", "NumberFire","NFL", "RTSports", "Walterfootball"),
                              pos = c("QB","RB","WR","TE","K","DST"))
    #scrape_nfl()

    #my_scrape[["QB"]] <- as_tibble(plyr::rbind.fill(my_scrape[["QB"]],NFL_DATA[["QB"]]))
    #my_scrape[["RB"]] <- as_tibble(plyr::rbind.fill(my_scrape[["RB"]],NFL_DATA[["RB"]]))
    #my_scrape[["WR"]] <- as_tibble(plyr::rbind.fill(my_scrape[["WR"]],NFL_DATA[["WR"]]))
    #my_scrape[["TE"]] <- as_tibble(plyr::rbind.fill(my_scrape[["TE"]],NFL_DATA[["TE"]]))
    #my_scrape[["K"]] <- as_tibble(plyr::rbind.fill(my_scrape[["K"]],NFL_DATA[["K"]]))
    #my_scrape[["DST"]] <- as_tibble(plyr::rbind.fill(my_scrape[["DST"]],NFL_DATA[["DST"]]))

    scrape_espn()
    my_scrape[["QB"]] <- as_tibble(plyr::rbind.fill(my_scrape[["QB"]],ESPN_DATA[["QB"]]))
    my_scrape[["RB"]] <- as_tibble(plyr::rbind.fill(my_scrape[["RB"]],ESPN_DATA[["RB"]]))
    my_scrape[["WR"]] <- as_tibble(plyr::rbind.fill(my_scrape[["WR"]],ESPN_DATA[["WR"]]))
    my_scrape[["TE"]] <- as_tibble(plyr::rbind.fill(my_scrape[["TE"]],ESPN_DATA[["TE"]]))
    my_scrape[["K"]] <- as_tibble(plyr::rbind.fill(my_scrape[["K"]],ESPN_DATA[["K"]]))
    my_scrape[["DST"]] <- as_tibble(plyr::rbind.fill(my_scrape[["DST"]],ESPN_DATA[["DST"]]))
    cat("",sep = "\n")

  }

  #########Fix data#########
  player_table1 <<- httr::GET("https://api.myfantasyleague.com/2021/export?TYPE=players&L=&APIKEY=&DETAILS=1&SINCE=&PLAYERS=&JSON=1") %>%
    httr::content() %>% `[[`("players") %>% `[[`("player") %>%
    purrr::map(tibble::as_tibble) %>%
    dplyr::bind_rows() %>%
    tidyr::extract(name, c("last_name", "first_name"), "(.+),\\s(.+)") %>%
    mutate_all(funs(gsub("(?![.-])[[:punct:]]", "", ., perl = T))) %>%
    dplyr::mutate(name = paste0(first_name," ",last_name)) %>%
    dplyr::select(id,name)

  for (p in list("QB","RB","WR","TE","K")) {
    for (j in 1:length(my_scrape[[p]][["id"]])){
      if (is.na(my_scrape[[p]][["id"]][[j]])) {
        my_scrape[[p]][["id"]][[j]] <- as.character(player_table1[match(my_scrape[[p]][["player"]][[j]], player_table1$name),1])
      }
    }
  }
  load_date <- format(Sys.time(), "%a %b %d %Y %I:%M %p")
  write.table(paste0("Data Updated as of ", load_date),paste0(getwd(),"/FantasyFootballApp/www/load_date.txt"),row.names=FALSE,sep="\t", quote = FALSE, col.names = FALSE)

  save(my_scrape,player_table,file = paste0(getwd(),"/Data/", vartype,"/ScrapeData/",varseason,"/Data Dump for ", Sys.Date(),".rdata"))
  rm(j,p,load_date)

  #########Data Export#########
  message("Starting Data Export")

    #Extract Postion Data for further analtysis
    #Clean and normaize data
    DataQB <- rbind(my_scrape[["QB"]]) %>% add_player_info() #QB Data
    DataRB <- rbind(my_scrape[["RB"]]) %>% add_player_info() #RB Data
    DataWR <- rbind(my_scrape[["WR"]]) %>% add_player_info() #WR Data
    DataTE <- rbind(my_scrape[["TE"]]) %>% add_player_info() #TE Data
    DataK <- rbind(my_scrape[["K"]]) %>% add_player_info() #K Data
    DataDST <- rbind(my_scrape[["DST"]]) %>% add_player_info()#DST Data


    DataQB$first_name <- paste(DataQB$first_name, DataQB$last_name)
    DataRB$first_name <- paste(DataRB$first_name, DataRB$last_name)
    DataWR$first_name <- paste(DataWR$first_name, DataWR$last_name)
    DataTE$first_name <- paste(DataTE$first_name, DataTE$last_name)
    DataK$first_name <- paste(DataK$first_name, DataK$last_name)
    DataDST$first_name <- paste(DataDST$first_name, DataDST$last_name)


    drops <<- c("last_name","src_id","player","team.y","number", "position.y",
                "tm","chg","player.1","pos","site_ci_low","site_ci_high", "ranks_ovr","bye","ranks_pos","z","note","status_game_opp",
                "z1","forecast","owner","games","fantasy_percent_owned",
                "rankings_proj","rankings_actual","passing_pick_six","opp","pass_rate",
                "displayname","esbid","gsisplayerid","site_season_pts", "site_pts","misc_fppg")
    keeps <<- c("id", "first_name", "position","position.x", "team.x", "age", "exp", "data_src",
                "pass_att", "pass_comp", "pass_inc", "pass_int", "pass_tds", "pass_yds",
                "rush_att", "rush_yds", "rush_tds",
                "rec", "rec_tgt", "rec_yds", "rec_tds", "reg_tds",
                "ret_yds", "ret_tds", "fumbles_lost", "fumbles_total", "punt_ret_yds", "kick_ret_yds", "two_pts")


    DataQB <- DataQB[ , (names(DataQB) %notin% drops)] %>% mutate_all(funs(ifelse(is.na(.), 0, .)))
    DataRB <- DataRB[ , (names(DataRB) %notin% drops)] %>% mutate_all(funs(ifelse(is.na(.), 0, .)))
    DataWR <- DataWR[ , (names(DataWR) %notin% drops)] %>% mutate_all(funs(ifelse(is.na(.), 0, .)))
    DataTE <- DataTE[ , (names(DataTE) %notin% drops)] %>% mutate_all(funs(ifelse(is.na(.), 0, .)))
    DataK <- DataK[ , (names(DataK) %notin% drops)] %>% mutate_all(funs(ifelse(is.na(.), 0, .)))
    DataDST <- DataDST[ , (names(DataDST) %notin% drops)] %>% mutate_all(funs(ifelse(is.na(.), 0, .)))

    names(DataQB)[2:4] <- c("name","team","position")
    names(DataRB)[2:4] <- c("name","team","position")
    names(DataWR)[2:4] <- c("name","team","position")
    names(DataTE)[2:4] <- c("name","team","position")
    names(DataK)[2:4] <- c("name","team","position")
    names(DataDST)[2:4] <- c("name","team","position")
    #Column Corrections

    DataQB <<- DataQB %>% add_column(pass_ypa = 0,pass_ypc =0, .before = "pass_tds") %>% dplyr::group_by(name,data_src) %>%
      mutate(rush_avg   = case_when(rush_att > 0 ~ round((rush_yds/rush_att),2), rush_att < 1 ~ 0, FALSE ~ 0),
             pass_yds_g = case_when(pass_comp > 0 ~ round((pass_yds/17),2), pass_comp < 1 ~ 0, FALSE ~ 0),
             pass_ypa   = case_when(pass_att > 0 ~ round((pass_yds/pass_att),2), pass_att < 1 ~ 0, FALSE ~ 0),
             pass_ypc   = case_when(pass_comp > 0 ~ round((pass_yds/pass_comp),2), pass_comp < 1 ~ 0, FALSE ~ 0),
             rush_tds   = case_when(rush_tds > 0 ~ rush_tds, rush_tds < 1 ~ rec_tds, FALSE ~ 0),
      ) %>% ungroup() %>% select(-rec_tds)

    DataRB <<- DataRB %>% add_column(rush_ypc = 0, rush_yds_g = 0, .before = "rush_tds") %>% dplyr::group_by(name,data_src) %>%
      mutate(rush_avg   = case_when(rush_att > 0 ~ round((rush_yds/rush_att),2), rush_att < 1 ~ 0, FALSE ~ 0),
             rush_yds_g = case_when(rush_att > 0 ~ round((rush_yds/17),2), rush_att < 1 ~ 0, FALSE ~ 0),
             rush_ypc   = case_when(rush_att > 0 ~ round((rush_yds/rush_att),2), rush_att < 1 ~ 0, FALSE ~ 0),
             rec_avg    = case_when(rec > 0 ~ round((rec_yds/rec),2), rec < 1 ~ 0, FALSE ~ 0),
             rec_yds_g  = case_when(rec > 0 ~ round((rec_yds/17),2), rec_yds < 1 ~ 0, FALSE ~ 0),
      ) %>% ungroup()

    DataWR <<- DataWR %>% add_column(rush_ypc = 0, rush_yds_g = 0, .before = "rush_tds") %>% dplyr::group_by(name,data_src) %>%
      mutate(rush_avg   = case_when(rush_att > 0 ~ round((rush_yds/rush_att),2), rush_att < 1 ~ 0, FALSE ~ 0),
             rush_yds_g = case_when(rush_att > 0 ~ round((rush_yds/17),2), rush_att < 1 ~ 0, FALSE ~ 0),
             rush_ypc   = case_when(rush_att > 0 ~ round((rush_yds/rush_att),2), rush_att < 1 ~ 0, FALSE ~ 0),
             rec_avg    = case_when(rec > 0 ~ round((rec_yds/rec),2), rec < 1 ~ 0, FALSE ~ 0),
             rec_yds_g  = case_when(rec > 0 ~ round((rec_yds/17),2), rec_yds < 1 ~ 0, FALSE ~ 0),
      ) %>% ungroup()

    DataTE <<- DataTE %>% dplyr::group_by(name,data_src) %>%
      mutate(rec_avg    = case_when(rec > 0 ~ round((rec_yds/rec),2), rec < 1 ~ 0, FALSE ~ 0),
             rec_yds_g  = case_when(rec > 0 ~ round((rec_yds/17),2), rec_yds < 1 ~ 0, FALSE ~ 0),
      ) %>% ungroup()

    #create detail data file
    DataAllOff <<- plyr::rbind.fill(DataQB,DataRB,DataWR,DataTE,DataDST) # Create table for all offense
    write.csv(DataAllOff, file = paste(getwd(),"/Data/", vartype,"/ScrapeData/",varseason,"/Raw Site Data.csv", sep=""), row.names=FALSE)
    QB <<- as.data.frame(DataQB)
    RB <<- as.data.frame(DataRB)
    WR <<- as.data.frame(DataWR)
    TE <<- as.data.frame(DataTE)
    K <<- as.data.frame(DataK)
    DST <<- as.data.frame(DataDST)

}