R/get_daily_client_properties.R

In chrisumphlett/reveneraR: Connect to Your 'Revenera' (Formerly 'Revulytics') Data

#' Get Daily Property Values for All Clients for a List of Product Ids
#'
#' Returns the list of daily client properties for all the client Ids
#' installed during a user provided date range for all the Product Ids. In
#' order to decrease how much is returned only the first date for a property
#' value is returned. For example, if Property A had value 1 for 3 days, then
#' value 2 for 2 days, then value 1 again on day 6, it will return the day 1
#' value of 1, day 4 value of 2, and day 6 value of 1.
#'
#' It is not recommended that your username be stored directly in your
#' code. There are various methods and packages available that are more
#' secure; this package does not require you to use any one in particular.
#'
#' This API call can only return 200 Client Ids at a time. It will take a
#' long time to execute if you have many Client Ids, as the function will
#' submit requests to the API repeatedly; this may even result in a timeout
#' error from the server. In order to provide data for troubleshooting
#' this function will write a message to the console after each call.
#' It is recommended that you divert the console output to a text file.
#' You can do this in multiple ways, including with the sink function (see
#' example for how to do this).
#'
#' For the same reason you are encouraged to break your request into
#' smaller chunks using the install dates and/or splitting up your
#' product Ids.
#'
#' @param rev_product_ids A vector of Revenera product id.
#' @param product_properties_df Data frame with available properties
#' for all product ids. Can obtain with the get_product_properties function.
#' @param desired_properties The property names of the metadata you want
#' to collect.
#' @param installed_start_date Date object for the starting date of
#' product installations.
#' @param installed_end_date Date object for the ending date of
#' product installations.
#' @param daily_start_date Date object for the starting date of desired
#' properties of the product.
#' @param daily_end_date Date object for the ending date of desired
#' properties of the product.
#' @param chatty The function can be chatty, sending a message to the console
#' for every iteration through a product Id. Many API calls may be required
#' and the console may get very long and it may slow down the execution.
#'
#' @import dplyr
#' @importFrom magrittr "%>%"
#' @importFrom purrr map_dfr map_dfc
#' @import httr
#' @import jsonlite
#' @import tidyr
#' @importFrom rlang .data
#'
#' @return Data frame with first date a property value appears until it changed
#' for each Client Id.
#'
#' @export
#'
#' @examples
#' \dontrun{
#' rev_user <- "my_username"
#' rev_pwd <- "super_secret"
#' logout(rev_user, rev_pwd)
#' Sys.sleep(30)
#' revenera_auth(rev_user, rev_pwd)
#' product_ids_list <- c("123", "456", "789")
#' product_properties <- get_product_properties(product_ids_list)
#' sink("output_filename.txt") # write out chatty messages to a file
#' sink(stdout(), type = "message")
#' daily_client_properties <- get_daily_client_properties(product_ids_list,
#'   product_properties, c("Property1", "Property2"), 
#'   installed_start_date = "01-01-2020", installed_end_date = "01-31-2020",
#'   daily_start_date = "01-01-2020", daily_end_date = "01-31-2020",
#'   chatty = TRUE
#' )
#' sink()
#' }
#'
get_daily_client_properties <- function(rev_product_ids,
                                        product_properties_df,
                                        desired_properties,
                                        installed_start_date,
                                        installed_end_date,
                                        daily_start_date, daily_end_date,
                                        chatty = FALSE) {
  
  #if an object with this name exists in the session, 
  #it will cause a problem further down
  if (exists("content_json")) {
    rm(content_json)
  }

  final_df <- data.frame()
  get_one_product_metadata <- function(x) {
    if (chatty) {
      message(paste0("Starting product id ", x))
    }

    custom_property_names <- product_properties_df %>%
      filter(
        .data$revenera_product_id == x,
        .data$property_friendly_name %in% desired_properties
      ) %>%
      select(.data$property_name) %>%
      pull()

    i <- 0

    keep_going <- TRUE

    while (keep_going == TRUE) {
      if (chatty) {
        message(paste0("iteration ", i))
      }
      
      i <- i + 1

      body <- paste0("{",
                     "\"startAtClientId\":",
                     jsonlite::toJSON(ifelse(exists("content_json"),
                                             content_json$nextClientId,
                                             NA_character_
                     ), auto_unbox = TRUE),
                     paste0(
                       ",\"globalFilters\":{\"dateInstalled\":",
                       "{\"type\":\"dateRange\",\"min\":\"",
                       installed_start_date,
                       "\",\"max\":\"",
                       installed_end_date,
                       "\"}},"
                     ),
                     paste0(
                       "\"retDailyData\":{\"startDate\":\"",
                       daily_start_date,
                       "\",\"stopDate\":\"",
                       daily_end_date,
                       "\",\"properties\":",
                       jsonlite::toJSON(array(c(custom_property_names)),
                                        auto_unbox = TRUE
                       ),
                       "}}"
                     ),
                     sep = ""
      )

      client_property_endpoint <- "reporting/clientPropertyList/"
      
      request <- httr::RETRY("POST",
        url = paste0(base_url, client_property_endpoint, x),
        add_headers(.headers = headers),
        body = body,
        encode = "json",
        times = 4,
        pause_min = 10,
        terminate_on = NULL,
        terminate_on_success = TRUE,
        pause_cap = 5
      )

      check_status(request)

      request_content <- httr::content(request, "text", encoding = "ISO-8859-1")
      content_json <- jsonlite::fromJSON(request_content, flatten = TRUE)
      
      # if there are not results for this product id, skip all of this
      if (length(content_json$result) > 0){
        if (chatty) {
          if (content_json$reachedEnd == "TRUE") {
            message("Reached end - no more clients")
          } else {
            message(paste0("nextClientId = ", content_json$nextClientId))
          }
        }
        
        build_data_frame <- function(c) {
          properties <- as.data.frame(content_json$result[c])
        }
        
        # build the data frame from json and unnest the daily property data
        product_df <- purrr::map_dfc(
          seq_len(length(content_json$result)),
          build_data_frame
        ) %>%
          tidyr::unnest(c("dailyData")) %>%
          dplyr::rename(client_id = .data$clientId, property_date = .data$date)
 
        # keep first date for each distinct property value
        client_df <- product_df %>%
          tidyr::pivot_longer(
            cols = -c("property_date", "client_id"),
            names_to = "property_name",
            values_to = "property_value"
          )  %>%
          dplyr::group_by(.data$client_id, .data$property_value) %>%
          dplyr::slice(which.min(as.Date(.data$property_date, "%Y-%m-%d"))) %>%
          dplyr::ungroup()
        
        final_df <- dplyr::bind_rows(final_df, client_df) %>%
          dplyr::mutate(revenera_product_id = x) %>%
          filter(!is.na(.data$property_value) 
                 & .data$property_value != "<NULL>")
        
      } else {
        if (chatty) {
          message("No results for this product id")
        }
      }

      keep_going <- ifelse(content_json$reachedEnd == "FALSE", TRUE, FALSE)
    }

    return(final_df)
  }

  all_products_df <- purrr::map_dfr(rev_product_ids, get_one_product_metadata)
  return(all_products_df)
}