H1

library(glue)        # formatting strings
library(lubridate)   # parsing datetime information
library(kableExtra)  # table formatting      
library(httr)        # or curl, httr, or rvest?
library(xml2)
library(tidyverse)   # powerful data processing
theme_set(theme_bw())# white background

setwd(here::here("/home/knut/code/git/_my/work/dis-docs/"))
setwd(here::here("guide/internal"))

url_mdfmt="https://doidb.wdc-terra.org/igsnoaip/oai?verb=ListMetadataFormats"

md_fmt <- xml2::read_xml(url_mdfmt,  as_html = FALSE,  options = "NOBLANKS") %>% 
    xml_ns_strip() %>% 
    xml2::xml_find_all(xpath = ".//metadataPrefix", ns = xml_ns(.)) %>% 
    xml2::xml_text(x = .)

url_doi="https://doidb.wdc-terra.org/igsnoaip/oai?"
url_sets="verb=ListSets"

url_catalogs <- sprintf("%s%s", url_doi, url_sets)
catalogs <- xml2::read_xml(url_catalogs) %>% 
  xml_ns_strip() 

setnames <- catalogs %>%   
  xml2::xml_find_all(xpath = ".//setName", ns = xml_ns(.)) %>% 
  xml_text() 

setspecs <- catalogs %>%   
  xml2::xml_find_all(xpath = ".//setSpec", ns = xml_ns(.)) %>% 
  xml_text() 
  
catalogs_df <- tibble(name = setnames, spec = setspecs) %>% 
  filter(str_detect(.$name, "(?i)Reference quality citations only", negate = TRUE) )

n <- 2009
n_years <-  lubridate::year(Sys.Date())
years <- tibble(year = seq(n, n_years))
dates_intervals <- years %>% 
  mutate(doy_first = sprintf("%s-01-01", year),
         doy_last =  sprintf("%s-12-31", year))

n_rows <- nrow(dates_intervals)
n_last <- n_rows - 2
dates_intervals %>% 
  rownames_to_column(var = "#") %>% 
  slice(c(seq(1,3), seq(n_last, n_rows))) %>% 
  knitr::kable(row.names = TRUE) %>%
  kableExtra::kable_styling(., bootstrap_options = "striped")

url_records = "verb=ListRecords"

url_records_tpl <- glue::glue("{url_doi}{url_records}&metadataPrefix=oai_dc")

http_requests <- catalogs_df %>% 
  crossing(dates_intervals) %>% 
  mutate(req = glue("{url_records_tpl}&from={doy_first}&until={doy_last}&set={spec}"))

#  purrr:partial
status_extract <- compose(httr::status_code, httr::GET)

find_all <- partial(xml2::xml_find_all, xpath = ".//resumptionToken/@completeListSize")


get_count <- compose(possibly(xml2::xml_text, otherwise = "0"), 
                       possibly(find_all,  otherwise = "0"),
                       possibly(xml_ns_strip, otherwise = "0"),
                       possibly(xml2::read_xml, otherwise = "0"))

outfile_rds <- "../../assets/other/data/igsns_per_year.RDS"
if (file.exists(outfile_rds)) {
  #igsns_per_year <- readRDS(outfile_rds)
  igsns_per_year <- readRDS(outfile_rds)
} else {
  igsns_per_year <- http_requests %>% 
    mutate(cnt = map(req, get_count)) 
  saveRDS(igsns_per_year, file = outfile_rds)
}

n_rows <- nrow(igsns_per_year)
n_last <- n_rows - 4

igsns_per_year %>% 
  arrange(year, name) %>% 
  rownames_to_column(var = "#") %>% 
  slice(c(seq(1,3), seq(n_last, n_rows))) %>% 
  knitr::kable(row.names = TRUE) %>%
  kableExtra::kable_styling(., bootstrap_options = "striped")

igsns_per_year_clean <- igsns_per_year %>% 
  arrange(spec, doy_last) %>% 
  mutate(cnt =  as.integer(cnt)) %>% 
  filter(str_detect(spec, "\\.", negate = TRUE)) %>% # remove "subcatalogs". 
  mutate(
         year2 = as.Date(glue("{year -1}-12-31")),
         spec2 = spec,
         spec2 = fct_reorder(spec2, -cnt, sum, na.rm = TRUE),
         catalog = as.factor(case_when(
    spec2 == "GEOAUS" ~ "GEO AUS",
    spec2 == "IEDA" ~ "SESAR",
    #spec2 == "GFZ" ~ "GFZ",
    TRUE ~ "(other)"
         )))
  
igsns_per_year_max <- igsns_per_year_clean %>% 
  arrange(spec, doy_last) %>% 
  filter(!is.na(cnt)) %>% 
  select(spec, year, cnt) %>% 
  group_by(spec) %>% 
  summarize(maxcnt = max(cnt, na.rm = TRUE)) %>% 
  ungroup() %>% 
  mutate(max_yearly = as.factor(case_when(
    maxcnt >= 10000 ~ ">= 10000",
    maxcnt >= 1000 ~ " >=  1000",
    TRUE ~ "<=  1000"
         )),
        max_yearly = fct_relevel(max_yearly, ">= 10000", " >=  1000", "<=  1000")) %>% 
  select(-maxcnt) 
  

igsns_per_year_clean <- igsns_per_year_clean %>% 
  inner_join(igsns_per_year_max, by = "spec")

igsns_per_year_sm <- igsns_per_year_clean %>% 
  filter(catalog == "(other)" ) 

p01_total <- igsns_per_year_clean %>% 
  filter(catalog != "(other)" ) %>% 
  filter(year >= 2009) %>% 
  ggplot(aes(year2, cnt, fill = catalog)) +
  geom_bar(stat = "identity", position = "dodge2") +
  #theme(legend.position = "none") +
  #annotate("text", x = as.Date("2017-12-31"), y = 3000, label = "turquoise = IGSNs of GEOFON records, 2013-2015\nred = IGSNs in all other catalogs") + 
  scale_x_date() + 
  scale_y_continuous(labels = scales::label_comma()) + 
  scale_fill_viridis_d() +
  labs(x = "Year since GFZ IGSN Allocation is in operation",
       y = "IGSN Count",
       title = "Count of IGSNs registered by 2 largest IGSN Allocation Agents, by year",
       subtitle = "Geo Australia, and SESAR, USA, www.geosamples.org\n")
p01_total

igsn_svg <- "img/igsn_svg_p01_total.svg"
ggsave(file = igsn_svg, plot = p01_total, width = 10, height = 8) 

p02_separate <- igsns_per_year_sm %>% 
  ggplot(aes(year2, cnt)) +
  geom_col() +
  scale_x_date() + 
  labs(x = "Year since GFZ IGSN Allocator is in operation",
       y = "Count",
       title = "Count of non-GFZ IGSNs registered by GFZ Dataservices, by year",
       subtitle = "Drill-Down: For all smaller catalogs separately.\n Largest catalogs are shown first") +
  facet_wrap(~spec2)
p02_separate

igsn_svg <- "img/igsn_svg_p02_separately_absolute.svg"
ggsave(file = igsn_svg, plot = p02_separate, width = 10, height = 8) 

p03_classifc <- igsns_per_year_sm %>% 
  ggplot(aes(year2, cnt, color = max_yearly, fill = max_yearly)) +
  geom_col() +
  theme(axis.text.x = element_text(angle=45, hjust = 1)) +
  labs(x = "Year since GFZ IGSN Allocator is in Operation",
       y = "Count",
       title = "Count of IGSNs registered with GFZ Dataservices, by year",
       subtitle = "Drill-Down: For each catalog separately. GEOAUS, SESAR catalogs: see extra plot.") +
  facet_wrap(~spec2, scales = "free_y")
p03_classifc

igsn_svg <- "img/igsn_svg_p03_with_classific.svg"
ggsave(file = igsn_svg, plot = p03_classifc, width = 10, height = 8)