In dbca-wa/etlTurtleNesting: Data ETL of Turtle Nesting Census Data from ODK Central to WAStD

```{css, echo=FALSE}

map {height: calc(100vh - 80px) !important;}

```r
# Libraries -------------------------------------------------------------------#
library(wastdr)
library(shiny)
library(flexdashboard)
library(leaflet)
library(fs)
library(here)
library(dplyr)
library(tidyr)
library(geojsonio)
library(geojsonsf)
library(janitor)
library(reactable)
library(RODBC)
library(DBI)

# Settings --------------------------------------------------------------------#
knitr::opts_chunk$set(echo = FALSE)

# Data ------------------------------------------------------------------------#
#
# WAMTRAM
w2_dir <- fs::dir_create(here::here("inst/data"))
w2_file <- here::here("inst/data/w2.rds")

w2_data_dl <- eventReactive(input$dlw2, {
  wastdr::wastdr_msg_info("Downloading new WAMTRAM data...")
  wastdr::download_w2_data(save=here::here("inst/w2.rds"))
  wastdr::wastdr_msg_success("Done!")
})

# w2_data <- reactive({w2_data <- readRDS(w2_file)})

if (fs::file_exists(w2_file)){ 
  wastdr::wastdr_msg_info("Reading saved WAMTRAM data...")
  w2_data <- readRDS(w2_file)
} else {
  wastdr::wastdr_msg_info("Downloading WAMTRAM data...")
  w2_data <- wastdr::download_w2_data(save=w2_file)
}



locations <- c("", sort(unique(w2_data$enc$location_code)))
places <- c("", sort(unique(w2_data$enc$place_code)))
w2_dl <- w2_data$downloaded_on

wastdr::wastdr_msg_info("Downloading WAStD Areas...")
areas_sf <- wastdr::wastd_GET("area") %>%
  magrittr::extract2("data") %>%
  geojsonio::as.json() %>%
  geojsonsf::geojson_sf()

areas_dl <- Sys.time()

areas <- areas_sf %>%
  dplyr::filter(area_type == "Locality") %>%
  dplyr::transmute(area_id = pk,
                   area_name = name,
                   w2_location_code = w2_location_code)

sites <- areas_sf %>%
  dplyr::filter(area_type == "Site") %>%
  dplyr::transmute(site_id = pk,
                   site_name = name,
                   w2_place_code = w2_place_code) %>%
  sf::st_join(areas)

sites_by_pc <- sites %>% 
  tidyr::separate_rows(w2_place_code)

enc_by_sites <- w2_data$enc %>% 
  dplyr::group_by(place_code) %>% 
  dplyr::tally(name = "observations") %>% 
  dplyr::ungroup()

homeless_places <- w2_data$sites %>% 
  dplyr::filter(is.na(site_latitude)) %>% 
  dplyr::mutate(search_areas_at=glue::glue(
    '<a href="https://wastd.dbca.wa.gov.au/admin/observations/area/?q={prefix}"',
    ' target="_" rel="external"><strong>{prefix}</strong></a>'),
    .before=code) %>% 
  dplyr::select(-site_datum, site_longitude, site_latitude) %>% 
  dplyr::left_join(enc_by_sites, by=c("code"="place_code")) %>% 
  dplyr::filter(observations > 0) %>% 
  dplyr::arrange(-observations) %>% 
  janitor::clean_names(case="title")

located_places <- w2_data$sites %>% 
  dplyr::filter(!is.na(site_latitude)) %>% 
  dplyr::mutate(search_areas_at=glue::glue(
    '<a href="https://wastd.dbca.wa.gov.au/admin/observations/area/?q={prefix}"',
    ' target="_" rel="external"><strong>{prefix}</strong></a>'),
    .before=code) %>% 
  dplyr::select(-site_datum, site_longitude, site_latitude) %>% 
  # dplyr::left_join(sites_by_pc, by=c("code"="w2_location_code")) %>% 
  dplyr::left_join(enc_by_sites, by=c("code"="place_code")) %>% 
  dplyr::filter(observations > 0) %>%
  dplyr::arrange(-observations) %>% 
  janitor::clean_names(case="title")

invalid_coords <- w2_data$enc %>% 
  dplyr::filter(
    longitude < -180 | 
      longitude > 180 | 
      latitude < -90 | 
      latitude > 90 |
      is.na(latitude) |
      is.na(longitude)
      )

unlikely_coords <- w2_data$enc %>% 
  dplyr::filter(
    longitude < 100 | longitude > 150 | latitude < -45 | latitude > 0)

w2_obs_wastd_sites <- w2_data$enc %>%
  dplyr::filter(
    longitude > -180, longitude < 180, latitude > -90, latitude < 90) %>%
  sf::st_as_sf(coords = c("longitude", "latitude"), crs = 4326) %>%
  sf::st_join(sites, left = TRUE)

Places

renderLeaflet({
  wastdr::map_wastd_wamtram_sites(areas, sites, w2_data$sites)
})

Observations {data-icon="fa-map-location-dot"}

Column {.sidebar}

Filter Observations

# actionButton("dlw2", "Download WAMTRAM", 
#              class = "btn btn-danger", icon=shiny::icon(name="download"))
inputPanel(
  selectInput("lc", label = "Location:", choices = locations),
  selectInput("pc", label = "Place:", choices = places),
  textInput("oid", label = "Observation ID:", value = "")
)

Place selected

selected_place <- reactive({w2_data$sites %>% dplyr::filter(code == input$pc)})
renderText({glue::glue("Lat {round(selected_place()$site_latitude, 5)}")})
renderText({glue::glue("Lon {round(selected_place()$site_longitude, 5)}")})

Column {data-width=150}

WAMTRAM

valueBox(w2_dl, icon = "fa-calendar", caption = "WAMTRAM")

WAStD

valueBox(areas_dl, icon = "fa-calendar", caption = "WAStD Areas")

Total Obs

total_obs <- reactive({
  ifelse(
    (is.null(input$pc) || input$pc == ""),
    w2_obs_wastd_sites %>% nrow(),
    w2_obs_wastd_sites %>% dplyr::filter(place_code == input$pc) %>% nrow()
  )
})
# renderText({total_obs()})
renderValueBox({
  valueBox(total_obs(), icon = "fa-map", caption = "Total Obs")
})

Obs inside sites

total_obs_inside <- reactive({
  ifelse(
    (is.null(input$pc) || input$pc == ""),
    w2_obs_wastd_sites %>% dplyr::filter(!is.na(site_name)) %>% nrow(),
    w2_obs_wastd_sites %>%
      dplyr::filter(place_code == input$pc, !is.na(site_name)) %>% nrow()
  )
})
renderValueBox({
  valueBox(total_obs_inside(), icon = "fa-map-marker", caption = "Obs in Sites")
})

Obs outside sites

total_obs_outside <- reactive({
  ifelse(
    (is.null(input$pc) || input$pc == ""),
    w2_obs_wastd_sites %>% dplyr::filter(is.na(site_name)) %>% nrow(),
    w2_obs_wastd_sites %>%
      dplyr::filter(place_code == input$pc, is.na(site_name)) %>% nrow()
  )
})
renderValueBox({
  valueBox(total_obs_outside(), 
           icon = "fa-question", 
           caption = "Obs not in Sites")
})

Pct inside sites

pct_obs <- reactive({round(100 * total_obs_inside() / total_obs(), 1)})
renderValueBox({

  valueBox(pct_obs(), 
           icon = "fa-percent", 
           caption = "Pct in Sites",
           color = ifelse(pct_obs() > 95, "success", "warning"))
})

Column

Map

renderLeaflet({
  wastdr::map_wamtram(
    w2_data, 
    location = input$lc, 
    place = input$pc, 
    obs_id = input$oid, 
    wa_sites = sites,
    l_height="calc(100vh - 80px)"
  )
})

Bad coords {data-icon="fa-danger"}

Impossible coordinates

reactable::renderReactable({
  reactable::reactable(
    invalid_coords, 
    sortable = TRUE, 
    filterable = TRUE, 
    searchable = TRUE,
    defaultColDef = colDef(html = TRUE)
    )
})

Unlikely coordinates

reactable::renderReactable({
  reactable::reactable(
    unlikely_coords, 
    sortable = TRUE, 
    filterable = TRUE, 
    searchable = TRUE,
    defaultColDef = colDef(html = TRUE)
    )
})

Places with Homes ™

Place statistics

renderValueBox({
  place_loc_rate <- round(100*(nrow(located_places) / nrow(w2_data$sites)), 0)
  flexdashboard::valueBox(
    value = place_loc_rate,
    caption = "% located",
    icon = "fa-area-chart",
    color = if (place_loc_rate <= 90) "warning" else "primary"
  )
})

Create WAStD sites for these WAMTRAM places

reactable::renderReactable({
  reactable::reactable(
    located_places, 
    sortable = TRUE, 
    filterable = TRUE, 
    searchable = TRUE,
    defaultColDef = colDef(html = TRUE)
    )
})

Places without Homes ™

Place statistics

renderValueBox({
  place_loc_rate <- round(100*(nrow(homeless_places) / nrow(w2_data$sites)), 0)
  flexdashboard::valueBox(
    value = place_loc_rate,
    caption = "% missing coordinates",
    icon = "fa-area-chart",
    color = if (place_loc_rate <= 90) "warning" else "primary"
  )
})

Update WAMTRAM with coordinates for these places

reactable::renderReactable({
  reactable::reactable(
    homeless_places, 
    sortable = TRUE, 
    filterable = TRUE, 
    searchable = TRUE,
    defaultColDef = colDef(html = TRUE)
    )
})

Protocols

Places

• Purpose: WAStD Areas of type Locality (WAMTRAM location) and Site (WAMTRAM Place) should correspond to all WAMTRAM Places within their bounds.
• WAStD Sites should be one nesting beach that is typically surveyed in one effort. Some nesting beaches are so large that they are split into sectors, which are surveyed individually.
• Some WAStD Sites can or should contain multiple WAMTRAM Places.
• We can split WAStD Sites later and re-link the WAMTRAM Places.

Red place markers are WAMTRAM sites without corresponding WAStD sites. For each red marker:

• Create a WAStD Area of type "Site".
• Create a polygon that buffers one nesting beach by ca 100m, include vegetation and shallow water.
• If there are other Sites close by, enable "WAStD Sites" from the layer menu.
• Go fullscreen for more convenient digitising if needed.
• Populate w2_location_code with exactly one W2 location code, taken from the red W2 Place marker. Code is all caps.
• Populate w2_place_code with any W2 Place's codes that are contained within. Separate multiple values with a whitespace. Codes are all caps.
• OK to ignore "survey length" and "roundtrip length" for now.
• If we ever need to report on surveys to a bunch of neighbouring Sites, create a WAStD Area with the Place's location code as w2_location_code. The Area should contain all Sites within.

WAStD Areas (Localities and Sites) are downloaded every time this dashboard is run.

Observations

• Purpose: Fix errors in W2 Observation coordinates.
• Select one place code.
• Review coordinates of misplaced observations (popup) against the coordinates of the selected place (sidebar).
• Most typos are immediately obvious, as the first two to three decimals of each coordinate should be the same or close for place and observation.
• Find observations through their observation ID in WAMTRAM and fix their coordinates.
• Leave a note in the observation comments about the coordinate change, e.g. "Lon 11123.456 changed to 123.456". This is the only change history we have.

To refresh WAMTRAM data, stop the dashboard, run the command below in the console, restart the R session (Ctrl+Shift+F10) then restart the dashboard (Ctrl+Shift+K).

w2_data <- wastdr::download_w2_data(save=here::here("inst/w2.rds"))

To load new wastdr functionality, run (when instructed to upgrade wastdr):

remotes::install_github("dbca-wa/wastdr")

Places without Home ™

Search WAStD for places at the same location code, create a WAStD Site for the WAMTRAM place with the missing coordinates, save, then use the centroid you can see in the WAStD Area list view as coordinates in WAMTRAM.

dbca-wa/etlTurtleNesting documentation built on Nov. 18, 2022, 8:03 a.m.