mDIS REST API access with R

library(tidyverse)    # data processing + plotting helpers
library(jsonlite)     # read/write JSON
library(httr)         # give R web browsing capabilities
library(kableExtra)   # nicer tables
theme_set(theme_bw()) # ggplot layout theme

my_c <- as_mapper(function(x, y) sprintf("%s=%s", x, y))
expedition <- c("Prees-2" = 4)
site <- c("Prees-2" = 5)
hole <- c("Prees-2" = 2)
qsp <- c(name = "core", "per-page" = 5000, "page" = 0, 
         sort = "id", 
         "filter[expedition_id]" = expedition, # JET 
         "filter[site_id]" = site,       # Prees
         "filter[hole_id]" = hole)       # main hole
qsparams <- map2_chr(names(qsp), qsp, my_c) %>% str_c(collapse = "&")

logindata <- jsonlite::fromJSON(txt = sprintf('{
  "username": "knb",
  "password": "knbpassword",
  "urlhost": "https://jet.rundis.com",
  "urlpaths": [
    "/api/v1/auth/login",
    "/api/v1/form?%s"
  ]
}', qsparams))

#logindata$password = Sys.getenv("MDIS_PW")
#jsonlite::toJSON(logindata)

# function according to the httr vignette
# returns simple s3 object for easier debugging
# used for login and HTTP-GETting data
mdis_api <- function(logindata, path) {
  url <- modify_url(logindata$urlhost, path = path)
  if (is.null(logindata$token)) {
    resp <- POST(url, ua = logindata$ua,
            body = list(username = logindata$username,
                        password = logindata$password))
  } else {
    resp <- GET(url,
                ua = logindata$ua,
                add_headers(
                  c(
                  Authorization = sprintf("Bearer %s", logindata$token))))
  }
  if (http_type(resp) != "application/json") {
    stop(str_c("API did not return json!", 
               "Invalid mDIS filter setting (= wrong query string params)?    ", 
               resp, 
               sep = "\n"), 
         call. = FALSE)
  }

  parsed <- jsonlite::fromJSON(content(resp, "text"), simplifyVector = FALSE)
  
  if (http_error(resp)) {
    stop(
      sprintf(
        "mDIS API request failed [%s]\n%s\n<%s>",
        status_code(resp),
        parsed[[1]],
        resp$url
      ),
      call. = FALSE
    )
  }
  
  # return simple S3 object, see comment below
  structure(
    list(
      content = parsed,
      path = path,
      response = resp
    ),
    class = "mdis_api"
  )  
}

# Rather than simply returning the response as a list,
# I think it’s a good practice to make a simple S3 object.
# That way you can return the response and parsed object,
# and provide a nice print method.
# This will make debugging later on much, much more pleasant.

print.mdis_api <- function(x, ...) {
  cat("<mDIS ", x$path, ">\n", sep = "")
  str(x$content)
  invisible(x)
}

# good practice: to identify yourself as user agent, ua
logindata$ua = user_agent("http://github.com/knbknb")

loggedin <- mdis_api(logindata, logindata$urlpaths[[1]])

logindata$token <- loggedin$content$token

parsed <- mdis_api(logindata, logindata$urlpaths[[2]])

cores <- as.list(parsed)$content$items %>%
  map(function(x) map(x, function(y) ifelse(is.null(y), NA, y)))

cores_df <- map_df(cores, as_tibble, .name_repair = "minimal")

cores_df <- cores_df %>%
  mutate(core_ondeck = as.POSIXct(core_ondeck))

#skimr::skim(cores_df)
glimpse(cores_df)

subtitle <- str_c("(fake data:) JET Core diameters (mm), Hole ", hole["Prees-2"], ", site ", site["Prees-2"], collapse = " ")
cores_df %>%
  group_by(core_diameter) %>%
  summarize(`max_depth (m)` = max(bottom_depth), n_Cores = n(), .groups = "keep")  %>%
  arrange(desc(core_diameter)) %>%
  ungroup() %>% 
  knitr::kable(caption = subtitle, col.names = names(.)) %>%
  kableExtra::kable_styling(bootstrap_options = c("striped"))

cores_df %>%
  count(analyst, sort = TRUE) %>%
  knitr::kable(caption = "Initials of active curators in the JET project", col.names = names(.)) %>%
  kableExtra::kable_styling(bootstrap_options = c("striped"))

table(as.Date(cores_df$core_ondeck), cores_df$core_loss_reason) %>%
  kable(caption = "When were JET cores lost? How many, and why?") %>%
  kableExtra::kable_styling(bootstrap_options = c("striped"))

cores_df %>% ggplot(aes(core_ondeck, core_recovery_pc/100, color=analyst)) +
  geom_jitter(alpha = 0.5) +
  geom_hline(yintercept = 1, color = "brown", linetype = 2, alpha = 0.5) +
  labs(title = "Fake Data (!): Prees-2 Core Recovery in Percent",
       subtitle = subtitle,
       y = "% Core Recovered",
       x = "Day of Recovery (2020)") +
  scale_y_continuous(labels = scales::percent_format(accuracy = NULL),  breaks = seq(0, 1.2, 0.1))