From c1d682e3ffe57c1da846e564398b11c44e509645 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 29 May 2026 08:01:25 -0500 Subject: [PATCH 01/32] updates to make env work --- environment.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 057a1989..c00bf875 100644 --- a/environment.yml +++ b/environment.yml @@ -1,8 +1,12 @@ name: dataretrieval channels: - conda-forge + - defaults dependencies: + - python>=3.10 + - pip - dataretrieval - - matplotlib + - matplotlib-base - geopandas + - pyqt prefix: /home/user/miniforge3/envs/dataretrieval From ca07d27355ba3a0a8a096b5aa0bfd772385c4c24 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 29 May 2026 08:24:39 -0500 Subject: [PATCH 02/32] try cleaning first --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7494678a..3f8077ba 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -136,7 +136,9 @@ pages: image: ${CI_REGISTRY_IMAGE}:latest stage: end before_script: - - conda env create -n dataretrieval -f environment.yml + - | + conda clean --all + conda env create -n dataretrieval -f environment.yml dependencies: - build-image - buildcheck From 86e2a4cc433e10783f15c25254f1ceccbbe8ba2f Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 29 May 2026 09:04:31 -0500 Subject: [PATCH 03/32] a few more CI tries --- .gitlab-ci.yml | 4 ---- docker/Dockerfile | 6 ++++++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3f8077ba..4ca1163c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -135,10 +135,6 @@ longtest: pages: image: ${CI_REGISTRY_IMAGE}:latest stage: end - before_script: - - | - conda clean --all - conda env create -n dataretrieval -f environment.yml dependencies: - build-image - buildcheck diff --git a/docker/Dockerfile b/docker/Dockerfile index d8997e5c..44afa219 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -17,6 +17,12 @@ RUN wget -O Miniforge3.sh "https://github.com/conda-forge/miniforge/releases/lat bash Miniforge3.sh -b -p "${HOME}/conda" && \ rm Miniforge3.sh +COPY environment.yml / +RUN conda env create -n ${CONDA_ENVIRONMENT_NAME} -f /environment.yml \ + && conda clean -a -y \ + && rm environment.yml + + # Necessary R libraries RUN apt-get update -qq && apt-get -y --no-install-recommends install \ r-cran-rcmdcheck \ From 694b76ce58011c4acff65a50cc45b42896af3070 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 29 May 2026 09:11:13 -0500 Subject: [PATCH 04/32] file paths --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 44afa219..288ef734 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -18,7 +18,7 @@ RUN wget -O Miniforge3.sh "https://github.com/conda-forge/miniforge/releases/lat rm Miniforge3.sh COPY environment.yml / -RUN conda env create -n ${CONDA_ENVIRONMENT_NAME} -f /environment.yml \ +RUN conda env create -n ${CONDA_ENVIRONMENT_NAME} -f environment.yml \ && conda clean -a -y \ && rm environment.yml From 87ae2d1099e8f62a6549e3b41f2bc1f1419cb57f Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 29 May 2026 09:22:44 -0500 Subject: [PATCH 05/32] another try --- .gitlab-ci.yml | 13 +++++-------- docker/Dockerfile | 2 +- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4ca1163c..6057c8ce 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -53,16 +53,13 @@ build-image: - name: ${DEVOPS_REGISTRY}usgs/docker:29-dind alias: docker script: - - echo ${CI_REGISTRY_PASSWORD} | docker login -u ${CI_REGISTRY_USER} --password-stdin $CI_REGISTRY + - echo "${CI_REGISTRY_PASSWORD}" | docker login -u "${CI_REGISTRY_USER}" --password-stdin $CI_REGISTRY - docker pull ${CI_REGISTRY_IMAGE}:latest || true - - cp *.tar.gz docker/ - - cd docker - - docker build - -t ${CI_REGISTRY_IMAGE}:latest - . + - docker build --cache-from ${CI_REGISTRY_IMAGE}:latest -t ${CI_REGISTRY_IMAGE}:latest -t ${CI_REGISTRY_IMAGE}:BUILD_${CI_COMMIT_SHORT_SHA} -f docker/Dockerfile . + # If this is building a git tag, also push the corresponding docker tag + - if [ -n "${CI_COMMIT_TAG}" ]; then docker tag "${CI_REGISTRY_IMAGE}:BUILD_${CI_COMMIT_SHORT_SHA}" "${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG}"; fi - docker push --all-tags ${CI_REGISTRY_IMAGE} - artifacts: - expire_in: 45 minutes + buildcheck: image: ${CI_REGISTRY_IMAGE}:latest diff --git a/docker/Dockerfile b/docker/Dockerfile index 288ef734..9671539c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -18,7 +18,7 @@ RUN wget -O Miniforge3.sh "https://github.com/conda-forge/miniforge/releases/lat rm Miniforge3.sh COPY environment.yml / -RUN conda env create -n ${CONDA_ENVIRONMENT_NAME} -f environment.yml \ +RUN conda env create -n ${CONDA_ENVIRONMENT_NAME} -f / environment.yml \ && conda clean -a -y \ && rm environment.yml From f48ac2a4e0cea1b2340173ae4b5a1e4a556e2147 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 29 May 2026 09:31:57 -0500 Subject: [PATCH 06/32] remove extra space --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 9671539c..44afa219 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -18,7 +18,7 @@ RUN wget -O Miniforge3.sh "https://github.com/conda-forge/miniforge/releases/lat rm Miniforge3.sh COPY environment.yml / -RUN conda env create -n ${CONDA_ENVIRONMENT_NAME} -f / environment.yml \ +RUN conda env create -n ${CONDA_ENVIRONMENT_NAME} -f /environment.yml \ && conda clean -a -y \ && rm environment.yml From 2f6143177afbccc22c457f41fa243fd0505e28df Mon Sep 17 00:00:00 2001 From: Joseph Zemmels Date: Fri, 29 May 2026 09:48:05 -0600 Subject: [PATCH 07/32] Try simpler Docker image build --- docker/Dockerfile | 88 +++++++++++++++++++++++------------------------ 1 file changed, 43 insertions(+), 45 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d8997e5c..f06309f6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,54 +1,52 @@ FROM code.chs.usgs.gov:5001/ctek/docker/r-lang/r-base:4.6 -# Change the name of this environment to something which pleases you, if you -# so please. But the name will not be relevant for most cases, as reticulate -# will be pointed to the environment no matter what it is named. -ARG CONDA_ENVIRONMENT_NAME=dataretrieval - -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - wget \ +# System dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 \ + python3-pip \ + # These might already come with r-cran-sf binary, commenting out for now: + # libgdal-dev \ + # libgeos-dev \ + # libproj-dev \ && rm -rf /var/lib/apt/lists/* - -ENV CONDA_DIR="/root/conda" -ENV PATH=$CONDA_DIR/bin:$PATH -RUN wget -O Miniforge3.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" && \ - bash Miniforge3.sh -b -p "${HOME}/conda" && \ - rm Miniforge3.sh +# Python dependencies +RUN pip3 install --no-cache-dir --break-system-packages \ + dataretrieval \ + matplotlib \ + geopandas \ + numpy \ + pandas \ + scipy \ + python-dateutil \ + requests -# Necessary R libraries -RUN apt-get update -qq && apt-get -y --no-install-recommends install \ - r-cran-rcmdcheck \ - r-cran-testthat \ - r-cran-covr \ - r-cran-reticulate \ - r-cran-curl \ - r-cran-lubridate \ - r-cran-xml2 \ - r-cran-httr2 \ - r-cran-whisker \ - r-cran-dplyr \ - r-cran-sf \ - r-cran-data.table \ - r-cran-jsonlite \ - r-cran-readr \ - r-cran-knitr \ - r-cran-rmarkdown \ - r-cran-dt \ - r-cran-leaflet \ - r-cran-readxl \ - r-cran-pkgdown \ - r-cran-ggplot2 \ - r-cran-tidyr \ - r-cran-purrr \ - && rm -rf /var/lib/apt/lists/* +# R dependencies +RUN apt-get update -qq && apt-get install -y --no-install-recommends \ + r-cran-reticulate \ + r-cran-curl \ + r-cran-lubridate \ + r-cran-xml2 \ + r-cran-httr2 \ + r-cran-dplyr \ + r-cran-sf \ + r-cran-ggplot2 \ + r-cran-tidyr \ + r-cran-purrr \ + r-cran-jsonlite \ + r-cran-knitr \ + r-cran-rmarkdown \ + r-cran-leaflet \ + r-cran-rcmdcheck \ + r-cran-testthat \ + r-cran-pkgdown \ + r-cran-covr \ + && rm -rf /var/lib/apt/lists/* -ENV RETICULATE_PYTHON=/root/conda/envs/${CONDA_ENVIRONMENT_NAME}/bin/python +ENV RETICULATE_PYTHON=/usr/bin/python3 COPY *.tar.gz /tmp/pkg.tar.gz - RUN set -eux; \ - R CMD INSTALL /tmp/pkg.tar.gz && \ - Rscript -e "library(dataRetrieval)" && \ - rm -f /tmp/*.tar.gz + R CMD INSTALL /tmp/pkg.tar.gz && \ + Rscript -e "library(dataRetrieval)" && \ + rm -f /tmp/*.tar.gz From c404e8bd74cc14888b82fc9ecc3fa2b6811a6fbf Mon Sep 17 00:00:00 2001 From: Joseph Zemmels Date: Fri, 29 May 2026 09:54:05 -0600 Subject: [PATCH 08/32] Add data.table and readr dependencies back --- docker/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index f06309f6..c231a02e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -37,6 +37,8 @@ RUN apt-get update -qq && apt-get install -y --no-install-recommends \ r-cran-knitr \ r-cran-rmarkdown \ r-cran-leaflet \ + r-cran-data-table \ + r-cran-readr \ r-cran-rcmdcheck \ r-cran-testthat \ r-cran-pkgdown \ From 25e66827cfd27f1dbd4619e6b64a4b75da1932ce Mon Sep 17 00:00:00 2001 From: Joseph Zemmels Date: Fri, 29 May 2026 09:58:51 -0600 Subject: [PATCH 09/32] Try datatable without hyphen --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index c231a02e..127f5dbc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -37,7 +37,7 @@ RUN apt-get update -qq && apt-get install -y --no-install-recommends \ r-cran-knitr \ r-cran-rmarkdown \ r-cran-leaflet \ - r-cran-data-table \ + r-cran-datatable \ r-cran-readr \ r-cran-rcmdcheck \ r-cran-testthat \ From 6437b91373afdd46112882faab4667db14526619 Mon Sep 17 00:00:00 2001 From: Joseph Zemmels Date: Fri, 29 May 2026 10:03:40 -0600 Subject: [PATCH 10/32] Use data.table in install --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 127f5dbc..391ef63a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -37,7 +37,7 @@ RUN apt-get update -qq && apt-get install -y --no-install-recommends \ r-cran-knitr \ r-cran-rmarkdown \ r-cran-leaflet \ - r-cran-datatable \ + r-cran-data.table \ r-cran-readr \ r-cran-rcmdcheck \ r-cran-testthat \ From 4275d1e7479e6b205ca34d5520021d0d55bf6110 Mon Sep 17 00:00:00 2001 From: Joseph Zemmels Date: Fri, 29 May 2026 10:11:29 -0600 Subject: [PATCH 11/32] Remove conda before_script in pages job --- .gitlab-ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7494678a..4ca1163c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -135,8 +135,6 @@ longtest: pages: image: ${CI_REGISTRY_IMAGE}:latest stage: end - before_script: - - conda env create -n dataretrieval -f environment.yml dependencies: - build-image - buildcheck From 47c823ae0c7b2dbc9c1dd64f84d0e7c633c06298 Mon Sep 17 00:00:00 2001 From: Joseph Zemmels Date: Fri, 29 May 2026 10:30:53 -0600 Subject: [PATCH 12/32] Add readxl and gridExtra to dockerfile, temporarily comment pkgdown build_site --- .gitlab-ci.yml | 8 ++++---- docker/Dockerfile | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4ca1163c..f2472194 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -139,10 +139,10 @@ pages: - build-image - buildcheck script: - - | - Rscript -e ' - pkgdown::build_site(override = list(destination = "public")) - file.copy(from = "./public/articles/logo.png", to = "./public/reference/logo.png")' + # - | + # Rscript -e ' + # pkgdown::build_site(override = list(destination = "public")) + # file.copy(from = "./public/articles/logo.png", to = "./public/reference/logo.png")' - quarto render artifacts: paths: diff --git a/docker/Dockerfile b/docker/Dockerfile index 391ef63a..feebe3a4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -43,6 +43,8 @@ RUN apt-get update -qq && apt-get install -y --no-install-recommends \ r-cran-testthat \ r-cran-pkgdown \ r-cran-covr \ + r-cran-readxl \ + r-cran-gridExtra \ && rm -rf /var/lib/apt/lists/* ENV RETICULATE_PYTHON=/usr/bin/python3 From b104a3da7a644b35a6b617c0d1ce645e5f786998 Mon Sep 17 00:00:00 2001 From: Joseph Zemmels Date: Fri, 29 May 2026 10:35:30 -0600 Subject: [PATCH 13/32] Fine, I guess we don't need gridExtra --- docker/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index feebe3a4..5581c68d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -44,7 +44,6 @@ RUN apt-get update -qq && apt-get install -y --no-install-recommends \ r-cran-pkgdown \ r-cran-covr \ r-cran-readxl \ - r-cran-gridExtra \ && rm -rf /var/lib/apt/lists/* ENV RETICULATE_PYTHON=/usr/bin/python3 From 63db3acc401e10cb20e9bf665c26413a05239e95 Mon Sep 17 00:00:00 2001 From: Joseph Zemmels Date: Fri, 29 May 2026 10:44:26 -0600 Subject: [PATCH 14/32] Try pointing quarto to python3 install --- .gitlab-ci.yml | 2 ++ docker/Dockerfile | 1 + 2 files changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f2472194..80b8414b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -138,6 +138,8 @@ pages: dependencies: - build-image - buildcheck + variables: + RETICULATE_PYTHON: /usr/bin/python3 script: # - | # Rscript -e ' diff --git a/docker/Dockerfile b/docker/Dockerfile index 5581c68d..170e4213 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -4,6 +4,7 @@ FROM code.chs.usgs.gov:5001/ctek/docker/r-lang/r-base:4.6 RUN apt-get update && apt-get install -y --no-install-recommends \ python3 \ python3-pip \ + python3-dev \ # These might already come with r-cran-sf binary, commenting out for now: # libgdal-dev \ # libgeos-dev \ From b350a90ecd0c2823839a49dc59ea47b7f34ae107 Mon Sep 17 00:00:00 2001 From: Joseph Zemmels Date: Fri, 29 May 2026 10:51:08 -0600 Subject: [PATCH 15/32] Add DT to dockerfile --- docker/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 170e4213..628b4913 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -45,6 +45,7 @@ RUN apt-get update -qq && apt-get install -y --no-install-recommends \ r-cran-pkgdown \ r-cran-covr \ r-cran-readxl \ + r-cran-dt \ && rm -rf /var/lib/apt/lists/* ENV RETICULATE_PYTHON=/usr/bin/python3 From 9c74fd27f8728f1b71c0f320a9b8b8a69a591ac7 Mon Sep 17 00:00:00 2001 From: Joseph Zemmels Date: Fri, 29 May 2026 11:02:24 -0600 Subject: [PATCH 16/32] Add pkgdown build_site back in --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 80b8414b..40414570 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -141,10 +141,10 @@ pages: variables: RETICULATE_PYTHON: /usr/bin/python3 script: - # - | - # Rscript -e ' - # pkgdown::build_site(override = list(destination = "public")) - # file.copy(from = "./public/articles/logo.png", to = "./public/reference/logo.png")' + - | + Rscript -e ' + pkgdown::build_site(override = list(destination = "public")) + file.copy(from = "./public/articles/logo.png", to = "./public/reference/logo.png")' - quarto render artifacts: paths: From f47795fa5542dac61511bb473ec758d3cb7cb752 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 29 May 2026 12:38:34 -0500 Subject: [PATCH 17/32] some updates to slides --- tutorials/basic_slides_deck.qmd | 220 +++++++++++--------------------- 1 file changed, 74 insertions(+), 146 deletions(-) diff --git a/tutorials/basic_slides_deck.qmd b/tutorials/basic_slides_deck.qmd index e814e022..7777b1b8 100644 --- a/tutorials/basic_slides_deck.qmd +++ b/tutorials/basic_slides_deck.qmd @@ -33,10 +33,16 @@ params: library(ggplot2) library(dplyr) library(reticulate) + py_require("dataretrieval") py_require("panda") py_require("matplotlib") +py_require("geopandas") +options(dplyr.summarise.inform = FALSE) + options(dplyr.summarise.inform = FALSE) +theme_set(theme_bw(base_size = 20)) +update_geom_defaults("point", list(size = 3)) evaluate_python <- params$run_python @@ -74,64 +80,13 @@ dt_me <- function( ## Introduction {background-image="images/hex_logo.png" background-size="15%" background-position="90% 90%" } -In this ~90 minute introduction, the goal is: - -- Introduce the modern `dataRetrieval` workflows. +The goal of these slides are to: -- The intended audience is someone: +- Introduce the modern USGS water data - - New to `dataRetrieval` +- Introduce `dataretrieval` (R and Python) - - Has some R experience - -::: footer - -::: - - -## RStudio Orientation - -By default will look like: - -![](images/default_rstudio.png) - -## RStudio Appearances - -Go to Tools -> Global Options -> Appearances to change style. - -![](images/apperences.png) - -## RStudio Orientation {.smaller} - -:::: {.columns} - -::: {.column width="30%"} - -1. Create scripts. - -2. See code run. - -3. See what variables are loaded - - - Click on a data frame to View - -4. Plots and more - -::: - -::: {.column width="70%"} - -![](images/dark_mode.png) -::: - -:::: - -::: footer - -::: - - -## dataRetrieval: R-package for US water data {.smaller} +## dataRetrieval: R or Python software for US water data {.smaller} :::: {.columns} @@ -218,27 +173,15 @@ from dataretrieval import waterdata ::: -## dataRetrieval: External Documentation - -![](images/documentation_1.png){width="1000" height="500"} +## External Documentation -::: footer - -::: - -## dataRetrieval: External Documentation - -![](images/documentation_2.png){width="1000" height="500"} - -::: footer - -::: +- R package: -## dataRetrieval: External Documentation +- Python package: -![](images/documentation_3.png){width="1000" height="500"} +- WDFN Blog: -## Documentation within R: function help pages {.smaller} +## Internal Documentation {.smaller} ::: {.panel-tabset} @@ -305,61 +248,49 @@ help(waterdata.get_daily) ### Challenge -1. Open RStudio +1. Open your preferred IDE (RStudio, VSCode, PyCharm, etc) or Jupyter notebook -2. Install `dataRetrieval`, `dplyr`, `ggplot2`, and `data.table` (if they are not already installed). +2. Install `dataRetrieval` (R) or `dataretrieval` (Python) if you haven't already. -3. Load `dataRetrieval` +3. Load `dataRetrieval` (R) / waterdata module in `dataretrieval` (Python) 4. Open the help file for the function `read_waterdata_daily` -5. Navigate to and find the list of function help files and explore some articles in "Additional Articles" +### R Solution: -### Solution: - - -```{r fig.height=7} +```{r} #| eval: false -install.packages(c("dataRetrieval", "dplyr", "ggplot2", "data.table")) +install.packages(c("dataRetrieval")) library(dataRetrieval) ?read_waterdata_daily ``` -::: - -::: footer - -::: - -## dataRetrieval Updates {background-image="images/hex_logo.png" background-size="15%" background-position="85% 80%" } - -Are you a seasoned `dataRetrieval` user? - -Here are resources for recent major changes: - -* [Changes to dataRetrieval](https://doi-usgs.github.io/dataRetrieval/articles/changes_slides.html) - -* [Water Data API Introduction](https://doi-usgs.github.io/dataRetrieval/articles/read_waterdata_functions.html) +### Python Solution: -* [Samples Data Introduction](https://doi-usgs.github.io/dataRetrieval/articles/samples_data.html) - -## What's New? {.smaller} +```{bash} +#| echo: true +#| eval: false +pip install dataretrieval +``` -There's been a lot of changes to `dataRetrieval` over the past year. If you'd like to see an overview of those changes, visit: [Changes to dataRetrieval](https://doi-usgs.github.io/dataRetrieval/articles/changes_slides.html) -Biggest changes: +```{python} +#| eval: false +from dataretrieval import waterdata -* NWIS servers will be shut down, so all `readNWIS` functions will eventually stop working +help(waterdata.get_daily) +``` -* `read_waterdata` functions are modern and should be used when possible -* The "USGS Water Data APIs" are the new home for USGS data +::: ::: footer ::: + + ## USGS Water Data API Token * The Water Data APIs limit how many queries a single IP address can make per hour @@ -377,18 +308,11 @@ Biggest changes: 1. Request a USGS Water Data API Token: -2. Save it in a safe place (KeePass or other password management tool) +2. Save it in a safe place (KeyPass or other password management tool) -3. Add it to your .Renviorn file as API_USGS_PAT. +3. Add it as environment variable -4. Restart R - -5. Check that it worked by running (you should see your token printed in the Console): - -```{r} -#| eval: false -Sys.getenv("API_USGS_PAT") -``` +4. Restart See next slide for a demonstration. @@ -396,9 +320,13 @@ See next slide for a demonstration. ::: -## USGS Water Data API Token: Example {.smaller} +## Water Data API Token: Example {.smaller} + +Let's pretend the token sent you was "abc123" -My favorite method to do add your token to .Renviron is to use the `usethis` package. Let's pretend the token sent you was "abc123": +::: {.panel-tabset} + +### R 1. Run in R: ```{r} @@ -409,37 +337,57 @@ usethis::edit_r_environ() 2. Add this line to the file that opens up: -```{r} -#| eval: false +``` API_USGS_PAT = "abc123" ``` -3. Save that file using the save button +3. Save that file 4. Restart R/RStudio. -5. Run after restarting R: +5. Check that it worked by running (you should see your token printed in the Console): ```{r} #| eval: false Sys.getenv("API_USGS_PAT") ``` -## USGS Water Data API Token: Example {.smaller .nostretch} +::: {.callout-note collapse="true"} +Your .Renviorn file should never be pushed to a public repository. +::: + +### Python -![](images/save_token.png){width="50%"} +1. Create a file in your working directory .env -After save and restart, check that it worked by running: +2. Add this line to the .env file: -```{r} +``` +API_USGS_PAT = "abc123" +``` +4. Restart your python session + +5. Check that it worked by running (you should see your token printed in the Console): + +```{python} +#| echo: true #| eval: false -Sys.getenv("API_USGS_PAT") +import os +os.getenv("API_USGS_PAT") +'abc123' ``` +::: {.callout-note collapse="true"} +Your .env file should never be pushed to a public repository. +::: + +::: + ::: footer ::: + ## USGS Basic Retrievals {.smaller} The USGS uses various codes for basic retrievals. These codes can have leading zeros, therefore they need to be a character surrounded in quotes ("00060"). @@ -483,29 +431,9 @@ We're going walk through 3 retrievals: * **Workflow 1**: Daily Data - - Uses the new USGS Water Data API - - - Modern data access point going forward - -* **Workflow 2**: Discrete Data - - - Uses new USGS Samples Data - - - Modern data access point going forward - -* **Workflow 3**: Join Daily and Discrete - -* **Workflow 4**: Continuous Data +* **Workflow 2**: Find Available Data - - Uses the new USGS Water Data API - - - Modern data access point going forward - -* **Workflow 5**: Join Continuous and Discrete -::: footer - -::: ## Workflow 1: Daily data for known site {.smaller} From de8dc4a31d3f429e60077c7025607906d5a7dec9 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 29 May 2026 15:25:40 -0500 Subject: [PATCH 18/32] more updates to slides --- tutorials/basic_slides_deck.qmd | 923 ++++++-------------------------- vignettes/basic_slides.Rmd | 2 +- vignettes/quick_slides.Rmd | 2 +- 3 files changed, 167 insertions(+), 760 deletions(-) diff --git a/tutorials/basic_slides_deck.qmd b/tutorials/basic_slides_deck.qmd index 7777b1b8..ae9180a8 100644 --- a/tutorials/basic_slides_deck.qmd +++ b/tutorials/basic_slides_deck.qmd @@ -40,9 +40,9 @@ py_require("matplotlib") py_require("geopandas") options(dplyr.summarise.inform = FALSE) -options(dplyr.summarise.inform = FALSE) theme_set(theme_bw(base_size = 20)) -update_geom_defaults("point", list(size = 3)) +update_geom_defaults("point", list(size = 3, color = "steelblue")) +options(ggplot2.discrete.colour = "viridis") evaluate_python <- params$run_python @@ -82,45 +82,99 @@ dt_me <- function( The goal of these slides are to: -- Introduce the modern USGS water data +- Introduce the modern USGS water data concepts + +- Introduce basic `dataretrieval` workflows (R and Python) -- Introduce `dataretrieval` (R and Python) +- Introduce additional topics that often come up -## dataRetrieval: R or Python software for US water data {.smaller} +## USGS Water Data Concepts {.smaller} :::: {.columns} ::: {.column width="50%"} +**USGS Water Data APIs** -**USGS Water Data APIs * ** - -- Surface water levels +- Continuous (e.g. 15-minute sensor data) -- Groundwater levels +- Daily (e.g. mean from continuous) -- Site metadata - -- Peak flows - -- Rating curves +- Monitoring Location Information + +- Time Series Information -- Discrete water-quality data +- Latest Daily/Continuous + +- Field Measurements ::: ::: {.column width="50%"} +**USGS Water Data APIs** + +- Peak Flows + +- Rating Curves + +- Discrete Water-Quality **Water Quality Portal (WQP) Data** -- Discrete water-quality data - -- USGS and non-USGS data +- Discrete water-quality data (USGS & others) ::: :::: -## Installation +::: {.callout-important} +## Each of these is has a different: + +* API endpoint +* `dataRetrieval` function +* Output format +::: + + +::: footer + +::: + +## USGS Basic Retrievals {.smaller} + +The USGS uses various codes for basic retrievals. These codes can have leading zeros, therefore they need to be a character surrounded in quotes ("00060"). + +* Site ID (often 8 or 15-digits) +* Parameter Code (5 digits) + + Full list: `read_waterdata_parameter_codes()` +* Statistic Code (for daily values) + + Full list: `read_metadata("statistic-codes")` + +## USGS Basic Retrievals Parameter and Statistic Codes + +Here are some examples of a few common codes: + + +```{r echo=FALSE, eval=TRUE} +library(knitr) + +df <- data.frame( + pCode = c("00060", "00065", "00010", "00400"), + shName = c("Discharge", "Gage Height", "Temperature", "pH") +) + +names(df) <- c("Parameter Code", "Short Name") + +df2 <- data.frame( + pCode = c("00001", "00002", "00003", "00008"), + shName = c("Maximum", "Minimum", "Mean", "Median") +) + +names(df2) <- c("Statistic Code", "Short Name") + +knitr::kable(list(df, df2)) +``` + +## Software Installation ::: {.panel-tabset} @@ -257,7 +311,7 @@ help(waterdata.get_daily) 4. Open the help file for the function `read_waterdata_daily` -### R Solution: +### R ```{r} #| eval: false @@ -266,7 +320,7 @@ library(dataRetrieval) ?read_waterdata_daily ``` -### Python Solution: +### Python ```{bash} #| echo: true @@ -388,158 +442,28 @@ Your .env file should never be pushed to a public repository. ::: -## USGS Basic Retrievals {.smaller} - -The USGS uses various codes for basic retrievals. These codes can have leading zeros, therefore they need to be a character surrounded in quotes ("00060"). - -* Site ID (often 8 or 15-digits) -* Parameter Code (5 digits) - + Full list: `read_waterdata_parameter_codes()` -* Statistic Code (for daily values) - + Full list: `read_metadata("statistic-codes")` - -## USGS Basic Retrievals Parameter and Statistic Codes - -Here are some examples of a few common codes: - - -```{r echo=FALSE, eval=TRUE} -library(knitr) - -df <- data.frame( - pCode = c("00060", "00065", "00010", "00400"), - shName = c("Discharge", "Gage Height", "Temperature", "pH") -) - -names(df) <- c("Parameter Code", "Short Name") - -df2 <- data.frame( - pCode = c("00001", "00002", "00003", "00008"), - shName = c("Maximum", "Minimum", "Mean", "Median") -) - -names(df2) <- c("Statistic Code", "Short Name") - -knitr::kable(list(df, df2)) -``` - - - -## Let's Go! {.smaller} - -We're going walk through 3 retrievals: - -* **Workflow 1**: Daily Data - -* **Workflow 2**: Find Available Data - - - -## Workflow 1: Daily data for known site {.smaller} - -Let's pull daily mean discharge data for site "USGS-0940550", getting all the data from the last year. - -::: {.panel-tabset} - -### R - -```{r} -#| message: true -library(dataRetrieval) -site <- "USGS-09405500" -pcode <- "00060" # Discharge -stat_cd <- "00003" # Mean - -df <- read_waterdata_daily( - monitoring_location_id = site, - parameter_code = pcode, - statistic_id = stat_cd, - time = "P365D" -) -nrow(df) -``` - -### Python - -```{python} -#| eval: !expr evaluate_python -from dataretrieval import waterdata - -site = "USGS-09405500" -pcode = "00060" # Discharge -stat_cd = "00003" # Mean - -df, md = waterdata.get_daily( - monitoring_location_id=site, - parameter_code=pcode, - statistic_id=stat_cd, - time="P365D", -) - -df.shape[0] -``` - -::: - -::: footer - -::: - -## Workflow 1: Look at Daily Data {.smaller} - -In RStudio, click on the data frame in the upper right Environment tab to open a Viewer. - -```{r} -#| echo: false -dt_me( - df |> - sf::st_drop_geometry(), - page_length = 3 -) -``` -::: footer -::: +## Water Data APIs: Initial Tips -## Workflow 1: Plot Daily Data +Use your "tab" key! ::: {.panel-tabset} ### R -Let's use `ggplot2` to visualize the data. - -```{r} -#| echo: true -#| output-location: column -library(ggplot2) -theme_set(theme_bw(base_size = 24)) -update_geom_defaults("point", list(size = 3, color = "steelblue")) -options(ggplot2.discrete.colour = "viridis") - -ggplot(data = df) + - geom_point(aes(x = time, y = value, color = approval_status)) -``` +![](images/autocomplete.png) ### Python -Let's use `matplotlib` to visualize the data. +![](images/autocomplete_python.png){width="60%"} -```{python} -#| echo: true -#| output-location: column -import matplotlib.pyplot as plt -import pandas as pd -plt.rcParams["font.size"] = 20 +### Jupyter Lab -levels, categories = pd.factorize(df["approval_status"]) +Shift + Tab: -fig, ax = plt.subplots() -scatter = ax.scatter(x=df.time, y=df.value, c=levels) -fig.legend(scatter.legend_elements()[0], categories, title="Status") -``` +![](images/jupyter_tab.png){width="60%"} ::: @@ -547,15 +471,9 @@ fig.legend(scatter.legend_elements()[0], categories, title="Status") ::: -## Water Data API Notes: Argument input - -Use your "tab" key! - -![](images/autocomplete.png) - -## Water Data API Notes: Arguments {.smaller} +## Water Data API Notes: Arguments -* When you look at the help file for the new functions, you’ll notice there are lots of possible inputs (arguments). +* When you look at the help file for the new functions, you’ll notice there are lots of possible inputs parameters. * You **DO NOT** need to (and should not!) specify **all** of these parameters. @@ -572,7 +490,7 @@ discharge <- read_waterdata_daily( ::: {.fragment} -::: {style="font-size: 75%;"} +::: {style="font-size: 50%;"} Since no list of sites or bounding box was defined, **ALL** the daily data in **ALL** the country with parameter code "00060" and statistic code "00003" will be returned. @@ -591,7 +509,7 @@ Since no list of sites or bounding box was defined, **ALL** the daily data in ** ::: {.column width="50%"} -The "time" argument has a few options: +Time parameters have a few options: * A single date (or date-time): "2024-10-01" or "2024-10-01T23:20:50Z" @@ -634,211 +552,90 @@ time = "PT12H" # past hours :::: -## Workflow 2: Discrete data for known site +## Let's Go! -Use your "tab" key! +* **Workflow 1**: Find Available Sites + +* **Workflow 2**: Find Available Data + +* **Workflow 3**: Get Latest Data + +* **Workflow 4**: Get All Data -![](images/autocomplete_samples.png) -## Workflow 2: Discrete data for known site {.smaller} +## Workflow 1: Find Available Sites -Let's get orthophosphate ("00660") data from the Shenandoah River at Front Royal, VA ("USGS-01631000"). +Let's get all the monitoring locations for Dane County, Wisconsin: ::: {.panel-tabset} ### R ```{r} -#| message: true -site <- "USGS-01631000" -pcode <- "00660" - -qw_data <- read_waterdata_samples( - monitoringLocationIdentifier = site, - usgsPCode = pcode, - dataType = "results", - dataProfile = "basicphyschem" +#| message: false +site_info <- read_waterdata_monitoring_location( + state_name = "Wisconsin", + county_name = "Dane County" ) -ncol(qw_data) ``` ### Python ```{python} #| eval: !expr evaluate_python -site = "USGS-01631000" -pcode = "00660" - -qw_data, md_qw = waterdata.get_samples( - monitoringLocationIdentifier = site, - usgsPCode = pcode, - service = "results", - profile = "basicphyschem", -) - -qw_data.shape[1] -``` - -::: - -That's a LOT of columns returned. We won't look at them here, but you can use `View` in RStudio to explore on your own. -::: footer - -::: - -## USGS Samples Data Notes: Data Types and Profiles - -* There are 2 arguments that dictate what kind of data is returned - - "dataType" defines what kind of data comes back - - "dataProfile" defines what columns from that type come back - -## Data Types and Profiles {.smaller} - -```{r} -#| echo: false -df <- tibble( - dataType = c( - "results", - "locations", - "activities", - "projects", - "organizations" - ), - Description = c( - "Results data and metadata for measures and observations matching your query", - "Find monitoring locations that have data matching your query", - "Information about the monitoring activities conducted that produced data", - "Information on the projects that have results matching your data query", - "Information about the organizations that have provided data that matches your query" - ), - dataProfile = c( - 'fullphyschem
basicphyschem
fullbio
basicbio
narrow
resultdetectionquantitationlimit
labsampleprep
count', - 'site
count', - 'sampact
actmetric
actgroup
ncount', - 'project
projectmonitoringlocationweight', - 'organization
count' - ) +site_info, md = waterdata.get_monitoring_locations( + state_name="Wisconsin", county_name="Dane County" ) - -dt_me(df, escape = FALSE, paging = FALSE) ``` -::: footer - ::: -## Workflow 2: Discrete data censoring {.smaller} - -Let's pull a few columns out and look at those: - -::: {.panel-tabset} - -### R +::: {.callout-note collapse="true"} +## Note on county names +`read_waterdata_monitoring_location` requires "County" in the county_name argument. You can check county names using: ```{r} -library(dplyr) - -qw_data_slim <- qw_data |> - select( - Date = Activity_StartDate, - Result_Measure, - DL_cond = Result_ResultDetectionCondition, - DL_val = DetectionLimit_MeasureA, - DL_type = DetectionLimit_TypeA - ) |> - mutate( - Result = if_else(DL_cond != "", DL_val, Result_Measure), - Detected = if_else(DL_cond != "", "Not Detected", "Detected") - ) |> - arrange(Detected) -``` - -* What is `|>`? It's a pipe! It says take 'this thing' and put it in 'that thing'. You'll also see `%>%` in code, it is also a pipe - they are basically the same. - -### Python - -```{python} -#| eval: !expr evaluate_python -import numpy as np - -qw_data_slim = ( - qw_data.rename( - columns={ - "Activity_StartDate": "Date", - "Result_ResultDetectionCondition": "DL_cond", - "DetectionLimit_MeasureA": "DL_val", - "DetectionLimit_TypeA": "DL_type", - } - )[["Date", "Result_Measure", "DL_cond", "DL_val", "DL_type"]] - .assign( - Result=lambda x: np.where( - x["DL_cond"].notna(), x["DL_val"], x["Result_Measure"] - ) - ) - .assign( - Detected=lambda x: np.where(x["DL_cond"].notna(), "Not Detected", "Detected") - ) - .sort_values(by="Detected", ascending=False) -) +#| eval: false +counties <- check_waterdata_sample_params(service = "counties") ``` - ::: ::: footer ::: -## Workflow 2: Discrete data censoring information {.smaller} +## site_info {.smaller .scrollable} ```{r} #| echo: false -dt_me(qw_data_slim, page_length = 8, font = "0.7em") +dt_me(site_info, 5, "0.6em") ``` ::: footer ::: -## Workflow 3: Join Discrete and Daily - -* One common workflow is to join discrete data with daily data. +## site_info_refined {.smaller} -* In this example, we will look at a site that measures both water quality parameters and has daily mean discharge. - -* We will use the `dplyr::left_join` to join the 2 data frames by a date. - -::: footer - -::: - -## Step 1: Get the data {.smaller} +Now that we've seen the whole data set, maybe we realize in the future we can ask for just stream sites, and we only really need a few of those columns: ::: {.panel-tabset} ### R ```{r} -site <- "USGS-04183500" -p_code_dv <- "00060" -stat_cd <- "00003" -p_code_qw <- "00665" -start_date <- "2015-07-03" -end_date <- "2025-07-03" - -qw_data <- read_waterdata_samples( - monitoringLocationIdentifier = site, - usgsPCode = p_code_qw, - activityStartDateLower = start_date, - activityStartDateUpper = end_date, - dataProfile = "basicphyschem" -) - -dv_data <- read_waterdata_daily( - monitoring_location_id = site, - parameter_code = p_code_dv, - statistic_id = stat_cd, - time = c(start_date, end_date) +#| message: true +site_info_refined <- read_waterdata_monitoring_location( + state_name = "Wisconsin", + county_name = "Dane County", + site_type = "Stream", + properties = c( + "monitoring_location_id", + "monitoring_location_name", + "drainage_area", + "geometry" + ) ) ``` @@ -846,500 +643,110 @@ dv_data <- read_waterdata_daily( ```{python} #| eval: !expr evaluate_python -site = "USGS-04183500" -p_code_dv = "00060" -stat_cd = "00003" -p_code_qw = "00665" -start_date = "2015-07-03" -end_date = "2025-07-03" - -qw_data, md_qw = waterdata.get_samples( - monitoringLocationIdentifier=site, - usgsPCode=p_code_qw, - activityStartDateLower=start_date, - activityStartDateUpper=end_date, - profile="basicphyschem", -) - -dv_data, md_dv = waterdata.get_daily( - monitoring_location_id=site, - parameter_code=p_code_dv, - statistic_id=stat_cd, - time=(start_date + "/" + end_date), +site_info_refined, md = waterdata.get_monitoring_locations( + state_name="Wisconsin", + county_name="Dane County", + site_type="Stream", + properties=[ + "monitoring_location_id", + "monitoring_location_name", + "drainage_area", + "geometry", + ], ) ``` - ::: ::: footer ::: -## Step 2: Join +## Map It with geometry {.smaller} ::: {.panel-tabset} -### R +### R: ggplot2 ```{r} -little_dv <- dv_data |> - select(time, Flow = value, monitoring_location_id) - -qw_data_joined <- qw_data |> - left_join(little_dv, by = c("Activity_StartDate" = "time")) -``` - - -### Python - -```{python} -#| eval: !expr evaluate_python -little_dv = dv_data.rename(columns={"value": "Flow"})[ - ["time", "Flow", "monitoring_location_id"] -] - -qw_data["Activity_StartDate"] = pd.to_datetime( - qw_data["Activity_StartDate"], format="%Y-%m-%d" -) - -qw_data_joined = pd.merge( - qw_data, little_dv, left_on="Activity_StartDate", right_on="time", how="left" -) -``` - -::: - -* "Activity_StartDate" (on the left side data frame) and "time" (on the right side data frame) need to be the same type (in this case, both are Date objects). - -::: footer - -::: - -## Step 2: Join (cont.) - -* You could join on multiple columns: - -::: {.panel-tabset} - -### R - -```{r} -#| eval: false -qw_data <- qw_data |> - left_join( - little_dv, - by = c( - "Activity_StartDate" = "time", - "Location_Identifier" = "monitoring_location_id" - ) - ) -``` - -See `dplyr` documentation for lots of joining options, but I find `left_join` my "go-to" for straightforward joins. - -### Python - -```{python} -#| eval: !expr evaluate_python -qw_data = pd.merge( - qw_data, - little_dv, - left_on=["Activity_StartDate", "Location_Identifier"], - right_on=["time", "monitoring_location_id"], - how="left", -) -``` - -::: - -::: footer - -::: - -## Step 3: Inspect - -Let's take a quick peak: - -::: {.panel-tabset} - -### R +#| output-location: default +library(ggplot2) -```{r} -#| output-location: column -ggplot(data = qw_data_joined) + - geom_point(aes(x = Flow, y = Result_Measure)) +ggplot(data = site_info_refined) + + geom_sf() ``` -### Python +### Python: matplotlib ```{python} #| eval: !expr evaluate_python -#| output-location: column -plt.figure() -plt.scatter(x=qw_data_joined.Flow, y=qw_data_joined.Result_Measure) -``` - - -::: - -## Exercise 2: Joins {.smaller} - -::: {.panel-tabset} - -### Challenge - -`dplyr` comes with some data sets. To look at them run: - -```{r} -library(dplyr) -band_members <- band_members -band_instruments <- band_instruments -``` - -1. Run that code and view the 2 data frames to see what they look like. - -2. Join the instruments to the "band_members" by name. - -3. Join the members to the "band_instruments" by name. - -### Solution: +import matplotlib.pyplot as plt +import geopandas as gpd +site_info_refined.plot() -```{r} -band_members |> - left_join(band_instruments, by = "name") - -band_instruments |> - left_join(band_members, by = "name") ``` - - ::: ::: footer ::: -## Workflow 4: Continuous data for known site - -* Continuous data is the high-frequency sensor data. - -* We'll look at Suisun Bay a Van Sickle Island NR Pittsburg CA ("USGS-11455508"), with parameter code "99133" which is Nitrate plus Nitrite. - -## Workflow 4: Continuous data for known site {.smaller} +## Map It: leaflet ::: {.panel-tabset} ### R -:::: {.columns} - -::: {.column width="65%"} - ```{r} -#| eval: false -site_id <- "USGS-11455508" -p_code_rt <- "99133" -start_date <- "2024-01-01" -end_date <- "2024-06-01" - -continuous_data <- read_waterdata_continuous( - monitoring_location_id = site_id, - parameter_code = p_code_rt, - time = c(start_date, end_date) -) -names(continuous_data) -``` - -::: +#| eval: true +#| output-location: slide +library(leaflet) +#default leaflet crs: +leaflet_crs <- "+proj=longlat +datum=WGS84" -::: {.column width="35%"} - -```{r} -#| results: markup -#| echo: false -options(width = 30) -site_id <- "USGS-11455508" -p_code_rt <- "99133" -start_date <- "2024-01-01" -end_date <- "2024-06-01" - -continuous_data <- read_waterdata_continuous( - monitoring_location_id = site_id, - parameter_code = p_code_rt, - time = c(start_date, end_date) -) -names(continuous_data) +leaflet( + data = site_info_refined |> + sf::st_transform(crs = leaflet_crs) +) |> + addProviderTiles("CartoDB.Positron") |> + addCircleMarkers(popup = ~monitoring_location_name, radius = 3, opacity = 1) ``` -::: - -:::: - ### Python -:::: {.columns} - -::: {.column width="65%"} +If you have `geopandas` installed, the function will return a GeoDataFrame with a geometry column containing the monitoring locations’ coordinates. You can use `gpd.explore()` to view your geometry coordinates on an interactive map. ```{python} #| eval: false -site_id = "USGS-11455508" -p_code_rt = "99133" -date_range = "2024-01-01/2024-06-01" - -continuous_data, md_cont = waterdata.get_continuous( - monitoring_location_id=site_id, parameter_code=p_code_rt, time=date_range -) -``` - -::: - -::: {.column width="35%"} -```{python} -#| eval: !expr evaluate_python -#| results: markup -#| echo: false -pd.set_option("display.width", 30) -site_id = "USGS-11455508" -p_code_rt = "99133" -date_range = "2024-01-01/2024-06-01" - -continuous_data, md_cont = waterdata.get_continuous( - monitoring_location_id=site_id, parameter_code=p_code_rt, time=date_range +site_info_refined.set_crs(crs="WGS84").explore( + marker_kwds={"radius": 3}, tiles="CartoDB.Positron" ) -continuous_data.columns -``` - -::: - -:::: - -::: - - -## Workflow 4: Inspect - -::: {.panel-tabset} - -### R - -```{r} -#| output-location: column -ggplot(data = continuous_data) + - geom_point(aes(x = time, y = value)) -``` - -### Python - -```{python} -#| eval: !expr evaluate_python -#| output-location: column -plt.figure() -plt.scatter(x=continuous_data.time, y=continuous_data.value) -``` - -::: - -## Workflow 5: Join Discrete and Continuous - -That same site also measures discrete Nitrate plus Nitrite, which is parameter code "00631". Let's first grab that data: - -::: {.panel-tabset} - -### R - -```{r} -#| message: true -discrete_data <- read_waterdata_samples( - monitoringLocationIdentifier = "USGS-11455508", - usgsPCode = "00631", - activityStartDateLower = start_date, - activityStartDateUpper = end_date, - dataProfile = "basicphyschem" -) -``` - -### Python - -```{python} -#| eval: !expr evaluate_python -discrete_data, md_qw = waterdata.get_samples( - monitoringLocationIdentifier = "USGS-11455508", - usgsPCode = "00631", - activityStartDateLower = "2024-01-01", - activityStartDateUpper = "2024-06-01", - profile = "basicphyschem" -) -``` - -::: - -## Workflow 5: Join Discrete and Continuous - -* We now want to join the **closest** continuous sensor time with the discrete sample time. - -* This is trickier than joining by exact matches. - -* `dplyr` has a way, but it's complicated if you want the absolute closest in either direction - -* Another package `data.table` has a slick way to get the closest matches - -## Workflow 5: Join Discrete and Continuous - -::: {.panel-tabset} - -### R - -```{r} -#| code-line-numbers: "1|2-3|5-10" -library(data.table) -setDT(discrete_data)[, join_date := Activity_StartDateTime] -setDT(continuous_data)[, join_date := time] - -closest_dt <- continuous_data[ - discrete_data, - on = .(join_date), - roll = "nearest" -] -closest_dt <- data.frame(closest_dt) ``` ::: -::: footer - -::: - -## Workflow 5: Inspect - -```{r} -#| output-location: column -ggplot(data = closest_dt) + - geom_point(aes(x = Result_Measure, y = value)) + - geom_abline() + - expand_limits(x = 0, y = 0) + - xlab("Discrete") + - ylab("Continuous") -``` - - -## Data Discovery - -The process for discovering data is a bit in flux with NWIS retiring. I expect a new process will be introduced soon. For now here are some options. - -1. `read_waterdata_ts_meta` discovers daily and continuous time series - -2. `summarize_waterdata_samples` discovers discrete data at specific monitoring locations - -The next slides will demo how to use those. - -## Data Discovery: Time Series {.smaller} - -::: {.panel-tabset} - -### R -```{r} -ts_available <- read_waterdata_ts_meta(monitoring_location_id = "USGS-04183500") -``` - -### Python - -```{python} -#| eval: false -ts_avail, ts_me = waterdata.get_time_series_metadata( - monitoring_location_id="USGS-04183500" -) -``` - -::: - -```{r} -#| echo: false -dt_me( - ts_available |> - sf::st_drop_geometry() |> - select( - parameter_name, - parameter_code, - statistic_id, - begin, - end, - computation_identifier - ), - page_length = 6 -) -``` - -::: footer - -::: - -## Data Discovery: Discrete {.smaller} - -::: {.panel-tabset} - -### R - -```{r} -discrete_available <- summarize_waterdata_samples( - monitoringLocationIdentifier = "USGS-04183500" -) -``` - - -::: - -```{r} -#| echo: false -dt_me( - discrete_available |> - select( - characteristicUserSupplied, - resultCount, - activityCount, - firstActivity, - mostRecentActivity - ), - page_length = 6 -) -``` - ::: footer ::: -## characteristicUserSupplied {.smaller} +## Workflow 2: Find Available Data -* characteristicUserSupplied can be an input to `read_waterdata_sample` +## Workflow 3: Get Latest Data -::: {.panel-tabset} +## Workflow 4: Get All Data -### R +## Workflow 5: Get Discrete Water Quality Data -```{r} -discrete1 <- read_waterdata_samples( - characteristicUserSupplied = "Phosphorus as phosphorus, water, unfiltered", - monitoringLocationIdentifier = "USGS-04183500" -) -nrow(discrete1) -``` +## USGS Samples Data Notes: Data Types and Profiles -### Python +* There are 2 arguments that dictate what kind of data is returned + - "dataType" defines what kind of data comes back + - "dataProfile" defines what columns from that type come back -```{python} -#| eval: !expr evaluate_python -discrete1, discrete1_me = waterdata.get_samples( - characteristicUserSupplied = "Phosphorus as phosphorus, water, unfiltered", - monitoringLocationIdentifier = "USGS-04183500" -) -discrete1.shape[1] -``` -::: ## More Information {.smaller} diff --git a/vignettes/basic_slides.Rmd b/vignettes/basic_slides.Rmd index fb543e0a..cb180289 100644 --- a/vignettes/basic_slides.Rmd +++ b/vignettes/basic_slides.Rmd @@ -12,7 +12,7 @@ editor_options: chunk_output_type: console --- -For a direct link to the slides, go [here](https://doi-usgs.github.io/dataRetrieval/tutorials/basic_slides_deck.html). Use the arrow keys to advance the slides. Or click the slide below then "f" for full screen, and "Esc" to escape full screen. +For a direct link to the slides, go [here](../tutorials/basic_slides_deck.html). Use the arrow keys to advance the slides. Or click the slide below then "f" for full screen, and "Esc" to escape full screen. ```{=html}