Bringing feature/new-rstudio-server up-to-date with master

.Rprofile

@@ -1,26 +1 @@
 source("renv/activate.R")
-
-# Set the repos using the renv.lock file
-renv_json <- jsonlite::read_json("renv.lock")
-renv_r_repos <- renv_json$R$Repositories
-
-# Extract the names
-repo_names <- purrr::flatten_chr(
-  purrr::map(renv_r_repos,
-             ~ .x$Name)
-)
-
-# Extract the URLs
-repo_urls <- purrr::flatten_chr(
-  purrr::map(renv_r_repos,
-             ~ .x$URL)
-)
-
-# Set the repo names
-names(repo_urls) <- repo_names
-
-# Set the options
-options(repos = repo_urls)
-
-# Remove all these objects
-rm(renv_json, renv_r_repos, repo_names, repo_urls)

.github/workflows/build-docker.yml

@@ -19,79 +19,157 @@
       - renv.lock
       - requirements.txt
       - current-modules.json
+      - .github/workflows/build-docker.yml
 
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+env:
+  REGISTRY_IMAGE: ccdl/training_rstudio
 jobs:
-  # This workflow contains a single job called "build"
   build:
-    # The type of runner that the job will run on
-    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: ubuntu-latest
+          - platform: linux/arm64
+            runner: ubuntu-24.04-arm
+    runs-on: ${{ matrix.runner }}
 
-    # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
+      - name: Prepare env variables
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
       - name: Check out the repo
         uses: actions/checkout@v5
 
       - name: Check that current-modules release-tag matches git tag
         if: startsWith(github.ref, 'refs/tags/')
         run: |
           GIT_TAG=${GITHUB_REF#refs/tags/}
-          MODULES_TAG=$(jq -r '.release-tag' current-modules.json)
+          MODULES_TAG=$(jq -r '."release-tag"' current-modules.json)
           if [ "$GIT_TAG" != "$MODULES_TAG" ]; then
             echo "Error: current-modules.json release-tag ($MODULES_TAG) does not match git tag ($GIT_TAG)"
             echo "Please update release-tag in current-modules.json to match the git tag (and modules, if needed), then update the GitHub release accordingly."
             exit 1
           fi
 
       - name: Load 1Password secrets
-        uses: 1password/load-secrets-action@v2
+        uses: 1password/load-secrets-action@v3
         with:
           export-env: true
         env:
           OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.TRAINING_OP_SERVICE_ACCOUNT_TOKEN }}
           DOCKER_USER: ${{ secrets.OP_DOCKER_USERNAME }}
           DOCKER_PASSWORD: ${{ secrets.OP_DOCKER_PASSWORD }}
-          ACTION_MONITORING_SLACK: ${{ secrets.OP_ACTION_MONITORING_SLACK }}
 
-      # Login to Dockerhub
       - name: Login to DockerHub
         uses: docker/login-action@v3
         with:
           username: ${{ env.DOCKER_USER }}
           password: ${{ env.DOCKER_PASSWORD }}
 
-      # set up Docker build
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
       - name: Docker metadata
         id: meta
         uses: docker/metadata-action@v5
         with:
-          images: ccdl/training_rstudio
+          images: ${{ env.REGISTRY_IMAGE }}
+
+      # Build Docker image, push only on push events
+      - name: Build Docker image
+        id: build
+        uses: docker/build-push-action@v6
+        with:
+          push: ${{ github.event_name == 'push' }}
+          platforms: ${{ matrix.platform }}
+          outputs: type=image,push-by-digest=true,name-canonical=true
+          labels: ${{ steps.meta.outputs.labels }}
+          tags: ${{ env.REGISTRY_IMAGE }}
+          cache-from: type=registry,ref=${{ env.REGISTRY_IMAGE }}:buildcache-${{ env.PLATFORM_PAIR }}
+          cache-to: type=registry,ref=${{ env.REGISTRY_IMAGE }}:buildcache-${{ env.PLATFORM_PAIR }},mode=max
+
+      - name: Export digest
+        if: github.event_name == 'push'
+        run: |
+          mkdir -p ${{ runner.temp }}/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "${{ runner.temp }}/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        if: github.event_name == 'push'
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ env.PLATFORM_PAIR }}
+          path: ${{ runner.temp }}/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  merge:
+    runs-on: ubuntu-latest
+    needs:
+      - build
+    # only merge the manifests and push on push events
+    if: github.event_name == 'push'
+    steps:
+      - name: Load 1Password secrets
+        uses: 1password/load-secrets-action@v3
+        with:
+          export-env: true
+        env:
+          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.TRAINING_OP_SERVICE_ACCOUNT_TOKEN }}
+          DOCKER_USER: ${{ secrets.OP_DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.OP_DOCKER_PASSWORD }}
+          ACTION_MONITORING_SLACK: ${{ secrets.OP_ACTION_MONITORING_SLACK }}
+
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: ${{ runner.temp }}/digests
+          pattern: digests-*
+          merge-multiple: true
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ env.DOCKER_USER }}
+          password: ${{ env.DOCKER_PASSWORD }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY_IMAGE }}
           # each github tag will create a matching tag on dockerhub,
           # with the most recent given the "latest" tag
           # the most recent push to master will get an "edge" tag
           tags: |
             type=ref,event=tag
             type=edge,branch=master
 
-      # Build Docker image, push only on push events
-      - name: Build Docker image
-        uses: docker/build-push-action@v5
-        with:
-          push: ${{ github.event_name == 'push' }}
-          tags: ${{ steps.meta.outputs.tags }}
-          cache-from: type=registry,ref=ccdl/training_rstudio:buildcache
-          cache-to: type=registry,ref=ccdl/training_rstudio:buildcache,mode=max
+      - name: Create manifest list and push
+        working-directory: ${{ runner.temp }}/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *)
+
+      - name: Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }}
 
       # If we have a failure, Slack us
       - name: Report failure to Slack
-        if: ${{ github.event_name == 'push' }}
+        if: ${{ failure() }}
         uses: ravsamhq/notify-slack-action@v2
         with:
           status: ${{ job.status }}
           notify_when: "failure"
           message_format: "Training build & push Docker workflow failed"
         env:
           SLACK_WEBHOOK_URL: ${{ env.ACTION_MONITORING_SLACK }}

.github/workflows/make-live.yml

@@ -14,20 +14,22 @@ jobs:
   make-live:
     # The type of runner that the job will run on
     runs-on: ubuntu-latest
-    container:
-      image: ccdl/training_rstudio:edge
 
     steps:
+      - name: Free disk space
+        run: |
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /usr/local/share/boost
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          # Print free disk space
+          df -h
+
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Configure git
-        run: |
-          git config --global --add safe.directory "$GITHUB_WORKSPACE"
-          git config --local user.email "actions@github.com"
-          git config --local user.name "GitHub Actions"
-
       - name: Load 1Password secrets
         uses: 1password/load-secrets-action@v2
         with:
@@ -43,15 +45,26 @@ jobs:
         env:
           AWS_DEFAULT_REGION: us-east-1
         run: |
-          aws s3 sync s3://ccdl-training-data/training-modules/ .
+          aws s3 sync s3://ccdl-training-data/training-modules/ . --no-progress
+
+      - name: Pull latest Docker image
+        run: |
+          docker pull ccdl/training_rstudio:edge
 
       - name: Render notebooks
         env:
           RENDER_RMD: ${{ github.event.inputs.rendering }}
-        run: bash scripts/render-live.sh
+        run: |
+          docker run --rm \
+            --mount type=bind,source="$GITHUB_WORKSPACE",target=/training-modules \
+            -w /training-modules \
+            -e RENDER_RMD \
+            ccdl/training_rstudio:edge \
+            bash scripts/render-live.sh
 
       # Make changes to pull request here
       - name: Create PR with rendered notebooks
+        id: cpr
         uses: peter-evans/create-pull-request@v6
         with:
           token: ${{ env.DOCS_BOT_GITHUB_TOKEN }}

.github/workflows/render-rmds.yml

@@ -6,20 +6,29 @@ on:
     branches:
       - master
     paths:
-      - '**.Rmd'
-      - '!**-live.Rmd' # don't trigger for live-only changes
-      - '!**/exercise*.Rmd' # or exercise notebooks
-      - '!**/setup/**.Rmd' # or setup notebooks
-      - 'scripts/make-live.R'
-      - 'scripts/render-live.sh'
+      - ".github/workflows/render-rmds.yml"
+      - "**.Rmd"
+      - "!**-live.Rmd" # don't trigger for live-only changes
+      - "!**/exercise*.Rmd" # or exercise notebooks
+      - "!**/setup/**.Rmd" # or setup notebooks
+      - "scripts/make-live.R"
+      - "scripts/render-live.sh"
 
 jobs:
   test-render:
     runs-on: ubuntu-latest
-    container:
-      image: ccdl/training_rstudio:edge
 
     steps:
+      - name: Free disk space
+        run: |
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /usr/local/share/boost
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          # Print free disk space
+          df -h
+
       - name: Checkout code
         uses: actions/checkout@v4
 
@@ -36,8 +45,16 @@ jobs:
         env:
           AWS_DEFAULT_REGION: us-east-1
         run: |
-          aws s3 sync s3://ccdl-training-data/training-modules/ .
+          aws s3 sync s3://ccdl-training-data/training-modules/ . --no-progress
 
+      - name: Pull latest Docker image
+        run: |
+          docker pull ccdl/training_rstudio:edge
 
       - name: Render notebooks
-        run: bash scripts/render-live.sh
+        run: |
+          docker run --rm \
+            --mount type=bind,source="$GITHUB_WORKSPACE",target=/training-modules \
+            -w /training-modules \
+            ccdl/training_rstudio:edge \
+            bash scripts/render-live.sh

CONTRIBUTING.md

@@ -215,7 +215,7 @@ In practice, this means that you will not need to add individual R packages to t
 To use the Docker image for development, pull from Docker Hub with:
 
 ```
-docker pull --platform linux/amd64 ccdl/training_rstudio:edge
+docker pull ccdl/training_rstudio:edge
 ```
 
 To run the container and mount a local volume, use the following from the root of this repository:

Dockerfile

@@ -1,5 +1,6 @@
 # Build salmon from source in a separate image
-FROM ubuntu:22.04 AS build
+# matching base image from https://github.com/rocker-org/rocker-versioned2/blob/master/dockerfiles/r-ver_4.5.2.Dockerfile
+FROM docker.io/library/ubuntu:noble AS build
 
 # Build dependencies
 RUN apt-get update -qq
@@ -15,6 +16,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
     libdeflate-dev \
     libisal-dev \
     liblzma-dev \
+    libzstd-dev \
     make \
     pkg-config \
     unzip \
@@ -29,7 +31,7 @@ RUN unzip awscliv2.zip
 RUN ./aws/install
 
 # Build salmon
-ARG SALMON_VERSION=1.10.1
+ARG SALMON_VERSION=1.10.3
 RUN curl -LO https://github.com/COMBINE-lab/salmon/archive/refs/tags/v${SALMON_VERSION}.tar.gz
 RUN tar xzf v${SALMON_VERSION}.tar.gz
 RUN mkdir salmon-${SALMON_VERSION}/build
@@ -45,12 +47,12 @@ RUN cd fastp-${FASTP_VERSION} && \
     make && make install
 
 # Main image with Biocconductor and other tools
-FROM bioconductor/bioconductor_docker:3.19 AS final
+FROM bioconductor/bioconductor_docker:3.22 AS final
 LABEL maintainer="ccdl@alexslemonade.org"
 
 WORKDIR /rocker-build/
 
-# Additonal dependencies for AWS runtime
+# Additional dependencies for AWS runtime
 RUN apt-get update -qq
 RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
     glibc-source \
@@ -66,21 +68,19 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
 
 # Python packages
 COPY requirements.txt requirements.txt
-RUN pip install -r requirements.txt
+RUN pip install -r requirements.txt --break-system-packages
 
 # Use renv for R packages
 WORKDIR /usr/local/renv
 ENV RENV_CONFIG_CACHE_ENABLED=FALSE
 RUN Rscript -e "install.packages('renv')"
 
-# Temporary fix for broken(?) RSamtools package
-RUN Rscript -e "install.packages('BiocManager'); BiocManager::install('Rsamtools')"
 
 COPY renv.lock renv.lock
-RUN Rscript -e "renv::restore()" \
-    rm -rf ~/.cache/R/renv && \
-    rm -rf /tmp/downloaded_packages && \
-    rm -rf /tmp/Rtmp*
+RUN Rscript -e "options(pkgType='binary'); renv::restore(repos = c(CRAN = 'https://packagemanager.posit.co/cran/__linux__/noble/latest'))" \
+    && rm -rf ~/.cache/R/renv \
+    && rm -rf /tmp/downloaded_packages \
+    && rm -rf /tmp/Rtmp*
 
 # copy aws, salmon, and fastp binaries from the build image
 COPY --from=build /usr/local/aws-cli/ /usr/local/aws-cli/