---- PURPOSE ----
Use one common structure to build up bundle. Except the query will be modified based on $$\code{type}$$
knitr::opts_chunk$set(echo = TRUE, comment=" ##", error=TRUE, collapse=TRUE) load_all()
## if you need: playlists <- readRDS(playlists, file=here("data","playlists.RDS"))
## Choose type types <- c("playlists", "videos", "comments", "playlist_count", "video_count") # REFERENCE # curl \ # 'https://youtube.googleapis.com/youtube/v3/playlistItems?part=snippet%2CcontentDetails&playlistId=PLbcglKxZP5PNx1CGmLKT68vKELqrhj4hY&fields=pageInfo&key=[YOUR_API_KEY]' \ # --header 'Authorization: Bearer [YOUR_ACCESS_TOKEN]' \ # --header 'Accept: application/json' \ # --compressed ## practice #type <- "videos" type <- "comments" ## to get all videos, for ONE playlist global_params <- initialize() global_params
#' returns query list generic_q <- function(){ ## query_defaults (starter values) if (!exists("query_defaults")) query_defaults <- get_typical_values() ## put it all togehter list( ## from global_params key = global_params$api$api_key, ## for videos: base_url = "https://www.googleapis.com/youtube/v3/playlistItems", part = "snippet", ## only change fields = "pageInfo.totalResults", #fields=paste(sep=",", "nextPageToken", # "items(id,snippet(title,description,publishedAt))"), ## from query_defaults (only what comments needs) maxResults = query_defaults$maxResults, channelId = query_defaults$channelId, playlistId = "PLbcglKxZP5PN3RCRdidmIxRrB8luufWIW", # gloria dehaven ## don't forget pageToken = NULL ) } query_params <- generic_q() query_params
# ============================= ## assume type = "comments" # ============================= query_params$base_url <- "https://www.googleapis.com/youtube/v3/commentThreads" query_params$part <- "snippet,replies" query_params$fields <- paste(sep = ", ", "nextPageToken", "items(snippet(topLevelComment(snippet(videoId,textDisplay))))" ) query_params$videoId = "Mec9sw1cJk8" # carter ## no playlistId query_params["playlistId"] <- NULL #query_params$playlistId = "PLlXfTHzgMRUIqYrutsFXCOmiqKUgOgGJ5" # Pavel Grinfeld, Linear Alg 3 ## and no channelId query_params["channelId"] <- NULL query_params ## spares ## query_params$playlistId <- "PLbcglKxZP5PMU2rNPBpYIzLTgzd5aOHw2" # playlistId <- "PLbcglKxZP5PN3RCRdidmIxRrB8luufWIW" # gloria dehaven # spare: # query_params$playlistId = "PLbcglKxZP5PN3RCRdidmIxRrB8luufWIW" # gloria dehaven # playlistId <- "PLlXfTHzgMRUIqYrutsFXCOmiqKUgOgGJ5" # Pavel Grinfeld, Linear Alg 3 # videoId <- "Mec9sw1cJk8" # carter #playlistId <- "PLbcglKxZP5PMZ7afIT7E2o9NwQIzqTI5l"
bundle <- list( url = query_params$base_url, config = global_params$config , query = query_params ) bundle$url bundle$config bundle$query
get_batch <- function(url = NULL, query = NULL, config = NULL) { r <- httr::GET(url = url, query = query, config = config, httr::verbose()) ## process resonse, return table ##process(x = videos, r=r, videosdb = videosdb) }
# ====================== ## assume type=videos # ====================== process <- function(r = NULL, type = NULL, db = NULL){ ## rbind is done aftert this call; not in this function json_content <- get_json(r) if (type == "videos") { db <- json_content$items$snippet nextPageToken <- json_content$nextPageToken list( nextPageToken = nextPageToken, db = db ) } if (type == "comments") { db <- json_content$items$snippet$topLevelComment$snippet db <- tibble::as_tibble(db) nextPageToken <- json_content$nextPageToken list( nextPageToken = nextPageToken, db = db ) } }
#### combine 1st batch + additional (if any) ```r ## run 1st time (may refresh stale token) db <- NULL r <- get_batch(url = bundle$url, query=bundle$query, config=bundle$config) r <- process(r=r, type="comments", db = db) db <- r$db nextPageToken <- r$nextPageToken ## Continue to get batches till null Token while (!is.null(nextPageToken)) { bundle$query$pageToken <- nextPageToken r <- get_batch(url = bundle$url, query=bundle$query, config=bundle$config) r <- process(r = r, type=type, db=db) db <- rbind(db, r$db) nextPageToken <- r$nextPageToken } ## return db } ##videos <- get_playlist_videos(playlistId) head(db, 2) View(db)
saveRDS(db, file=here::here("data/videosdb.RDS"))
cleanup videos
t <- videos %>% dplyr::select(title, position) %>% dplyr::arrange(title) t str(t)
here() file <- "rmd/012_get_videos.Rmd" file <- basename(file) dir="rmd" jimTools::ren_pdf(file,dir) jimTools::ren_github(file, dir)
Add the following code to your website.
For more information on customizing the embed code, read Embedding Snippets.