Migrate from pip to uv and upgrade Python 3.9 to 3.11

.github/workflows/build_test_ds_arm.yaml

@@ -53,8 +53,12 @@ jobs:
     needs: detect-changes
     runs-on: ubuntu-22.04-arm-aws
     steps:
-      - name: Fix workspace permissions
-        run: docker run --rm -v ${{ github.workspace }}:/workspace alpine chown -R $(id -u):$(id -g) /workspace || true
+      - name: Pre-cleanup workspace and Docker
+        continue-on-error: true
+        run: |
+          sudo rm -rf ${{ github.workspace }}/* || true
+          sudo rm -rf ${{ github.workspace }}/.* || true
+          docker system prune -af || true
 
       - name: Checkout code
         uses: actions/checkout@v4
@@ -88,18 +92,18 @@ jobs:
 
       - name: Save Docker images
         run: |
-          docker save awiciroh/datastream:latest-arm64 | gzip > /tmp/datastream-arm64.tar.gz
+          docker save awiciroh/datastream:latest-arm64 | gzip > ${{ runner.temp }}/datastream-arm64.tar.gz
 
       - name: Upload datastream image
         uses: actions/upload-artifact@v4
         with:
           name: datastream-arm64
-          path: /tmp/datastream-arm64.tar.gz
+          path: ${{ runner.temp }}/datastream-arm64.tar.gz
           retention-days: 1
 
       - name: Clean up local tar files
         if: always()
-        run: rm -f /tmp/datastream-arm64.tar.gz
+        run: rm -f ${{ runner.temp }}/datastream-arm64.tar.gz
 
 
   test-arm64:
@@ -155,33 +159,30 @@ jobs:
     name: "${{ matrix.test-suite.name }}"
 
     steps:
-      - name: Fix workspace permissions
-        run: docker run --rm -v ${{ github.workspace }}:/workspace alpine chown -R $(id -u):$(id -g) /workspace || true
-
+      - name: Pre-cleanup workspace and Docker
+        continue-on-error: true
+        run: |
+          sudo rm -rf ${{ github.workspace }}/* || true
+          sudo rm -rf ${{ github.workspace }}/.* || true
+          docker system prune -af || true
 
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Configure AWS
-        run: |
-          aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws configure set region us-east-1
-
       - name: Download Docker image
         uses: actions/download-artifact@v4
         with:
           name: datastream-arm64
-          path: /tmp/artifacts
+          path: ${{ runner.temp }}/artifacts
 
       - name: Verify artifact and load
         run: |
           set -euo pipefail
-          ls -lh /tmp/artifacts
+          ls -lh ${{ runner.temp }}/artifacts
 
           # Load gzipped image
           for i in 1 2 3; do
-            if gunzip -c /tmp/artifacts/datastream-arm64.tar.gz | docker load; then
+            if gunzip -c ${{ runner.temp }}/artifacts/datastream-arm64.tar.gz | docker load; then
               echo "Successfully loaded image on attempt $i"
               docker images | grep datastream
               break

.github/workflows/build_test_ds_x86.yaml

@@ -22,38 +22,67 @@ permissions:
   security-events: write
 
 jobs:
+  detect-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      deps-changed: ${{ steps.changes.outputs.deps }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Check for deps Dockerfile changes
+        id: changes
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            DIFF=$(git diff --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }})
+          elif [ "${{ github.event_name }}" == "push" ]; then
+            DIFF=$(git diff --name-only ${{ github.event.before }} ${{ github.sha }})
+          else
+            DIFF=$(git diff --name-only origin/main...HEAD)
+          fi
+
+          if echo "$DIFF" | grep -q "docker/Dockerfile.datastream-deps"; then
+            echo "deps=true" >> $GITHUB_OUTPUT
+          else
+            echo "deps=false" >> $GITHUB_OUTPUT
+          fi
+
   build-x86:
+    needs: detect-changes
     runs-on: ubuntu-latest
     steps:
       - name: Pre-cleanup workspace
         continue-on-error: true
         run: |
           sudo rm -rf ${{ github.workspace }}/* || true
           sudo rm -rf ${{ github.workspace }}/.* || true
-          
+
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
       - name: Configure AWS
         run: |
           aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws configure set region us-east-1
-
-      - name: Build datastream-deps
+
+      - name: Build datastream-deps (only if Dockerfile changed)
+        if: needs.detect-changes.outputs.deps-changed == 'true'
         run: |
           cd docker
-          export TAG=latest-x86
-          docker compose -f docker-compose.yml build datastream-deps
-
+          TAG=latest-x86 docker compose -f docker-compose.yml build datastream-deps
+
+      - name: Pull datastream-deps from Docker Hub (if not built)
+        if: needs.detect-changes.outputs.deps-changed != 'true'
+        run: |
+          docker pull awiciroh/datastream-deps:latest-x86
+
       - name: Build datastream
         run: |
           cd docker
-          export TAG=latest-x86
-          docker compose -f docker-compose.yml build datastream
+          TAG=latest-x86 docker compose -f docker-compose.yml build datastream
 
       - name: List built images
         run: docker images | grep datastream

.github/workflows/integration_ds_fp_arm.yaml

@@ -44,6 +44,8 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
       - name: Setup Terraform
         uses: hashicorp/setup-terraform@v3
       - name: Configure AWS
@@ -60,18 +62,29 @@ jobs:
         run: |
           if [ "${{ github.event_name }}" == "workflow_dispatch" ] && [ -n "${{ inputs.branch_name }}" ]; then
             BRANCH_NAME="${{ inputs.branch_name }}"
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            BRANCH_NAME="${{ github.head_ref }}"
           else
             BRANCH_NAME="${{ github.ref_name }}"
           fi
 
           sed -i "s|\${BRANCH_NAME}|$BRANCH_NAME|g" .github/executions/fp_ds_test_execution_arm.json
           sed -i "s|\${DS_TAG}|$DS_TAG|g" .github/executions/fp_ds_test_execution_arm.json
           sed -i "s|\${FP_TAG}|$FP_TAG|g" .github/executions/fp_ds_test_execution_arm.json
-          if [ "${{ github.event_name }}" != "workflow_dispatch" ] && ! git diff --name-only HEAD~1 HEAD | grep -q '^docker/Dockerfile\.datastream-deps$'; then
+
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            DIFF=$(git diff --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }})
+          elif [ "${{ github.event_name }}" == "push" ]; then
+            DIFF=$(git diff --name-only ${{ github.event.before }} ${{ github.sha }})
+          else
+            DIFF=$(git diff --name-only origin/main...HEAD)
+          fi
+
+          if [ "${{ github.event_name }}" != "workflow_dispatch" ] && ! echo "$DIFF" | grep -q '^docker/Dockerfile\.datastream-deps$'; then
             sed -i "/docker-compose\.yml build datastream-deps/d" .github/executions/fp_ds_test_execution_arm.json
           fi
-          
-          if [ "${{ github.event_name }}" != "workflow_dispatch" ] && ! git diff --name-only HEAD~1 HEAD | grep -q '^docker/Dockerfile\.datastream$'; then
+
+          if [ "${{ github.event_name }}" != "workflow_dispatch" ] && ! echo "$DIFF" | grep -q '^docker/Dockerfile\.datastream$'; then
             sed -i "/docker-compose\.yml build datastream/d" .github/executions/fp_ds_test_execution_arm.json
           fi
 

.github/workflows/integration_ds_fp_x86.yaml

@@ -37,12 +37,8 @@ jobs:
     steps:
     - name: Checkout code
       uses: actions/checkout@v4
-
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@v3
-
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
+      with:
+        fetch-depth: 0
 
     - name: Login to Docker Hub
       uses: docker/login-action@v3
@@ -57,10 +53,23 @@ jobs:
         aws configure set region us-east-1
 
     - name: Build docker containers
-      run : |        
+      run : |
         if [ -z "${{ inputs.fp_tag }}" ]; then
           echo "FP_TAG input not provided, building datastream containers..."
-          TAG=latest-x86 docker compose -f docker/docker-compose.yml build datastream-deps
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            DIFF=$(git diff --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }})
+          elif [ "${{ github.event_name }}" == "push" ]; then
+            DIFF=$(git diff --name-only ${{ github.event.before }} ${{ github.sha }})
+          else
+            DIFF=$(git diff --name-only origin/main...HEAD)
+          fi
+          if echo "$DIFF" | grep -q "docker/Dockerfile.datastream-deps"; then
+            echo "Deps Dockerfile changed, building deps..."
+            TAG=latest-x86 docker compose -f docker/docker-compose.yml build datastream-deps
+          else
+            echo "Deps Dockerfile unchanged, pulling from Docker Hub..."
+            docker pull awiciroh/datastream-deps:latest-x86
+          fi
           TAG=latest-x86 docker compose -f docker/docker-compose.yml build datastream
         else
           echo "FP_TAG input provided (${{ inputs.fp_tag }}), skipping datastream build and using existing images..."

.github/workflows/test_datastream_ngiab.yaml

@@ -32,13 +32,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout code
-      uses: actions/checkout@v2  
-
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@v3
-
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3      
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
 
     - name: Configure AWS
       run: |
@@ -48,9 +44,25 @@ jobs:
         export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}
         export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
-    - name: Build docker containers
-      run : |        
-        docker compose -f docker/docker-compose.yml build datastream
+    - name: Build or pull datastream-deps
+      run: |
+        if [ "${{ github.event_name }}" == "pull_request" ]; then
+          DIFF=$(git diff --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }})
+        elif [ "${{ github.event_name }}" == "push" ]; then
+          DIFF=$(git diff --name-only ${{ github.event.before }} ${{ github.sha }})
+        else
+          DIFF=$(git diff --name-only origin/main...HEAD)
+        fi
+        if echo "$DIFF" | grep -q "docker/Dockerfile.datastream-deps"; then
+          echo "Deps Dockerfile changed, building..."
+          docker compose -f docker/docker-compose.yml build datastream-deps
+        else
+          echo "Deps Dockerfile unchanged, pulling from Docker Hub..."
+          docker pull awiciroh/datastream-deps:latest
+        fi
+
+    - name: Build datastream
+      run: docker compose -f docker/docker-compose.yml build datastream
 
     - name: Base test and NWM_RETRO_V3
       run: |        

.github/workflows/test_datastream_options.yaml

@@ -24,16 +24,32 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - uses: docker/setup-qemu-action@v3
-      - uses: docker/setup-buildx-action@v3
-
+        with:
+          fetch-depth: 0
       - name: Configure AWS
         run: |
           aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws configure set region us-east-1
 
-      - name: Build docker containers
+      - name: Build or pull datastream-deps
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            DIFF=$(git diff --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }})
+          elif [ "${{ github.event_name }}" == "push" ]; then
+            DIFF=$(git diff --name-only ${{ github.event.before }} ${{ github.sha }})
+          else
+            DIFF=$(git diff --name-only origin/main...HEAD)
+          fi
+          if echo "$DIFF" | grep -q "docker/Dockerfile.datastream-deps"; then
+            echo "Deps Dockerfile changed, building..."
+            docker compose -f docker/docker-compose.yml build datastream-deps
+          else
+            echo "Deps Dockerfile unchanged, pulling from Docker Hub..."
+            docker pull awiciroh/datastream-deps:latest
+          fi
+
+      - name: Build datastream
         run: docker compose -f docker/docker-compose.yml build datastream
 
       - name: Setup test environment

.github/workflows/test_datastream_options_forcings_sources.yaml

@@ -24,16 +24,32 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - uses: docker/setup-qemu-action@v3
-      - uses: docker/setup-buildx-action@v3
-
+        with:
+          fetch-depth: 0
       - name: Configure AWS
         run: |
           aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws configure set region us-east-1
 
-      - name: Build docker containers
+      - name: Build or pull datastream-deps
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            DIFF=$(git diff --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }})
+          elif [ "${{ github.event_name }}" == "push" ]; then
+            DIFF=$(git diff --name-only ${{ github.event.before }} ${{ github.sha }})
+          else
+            DIFF=$(git diff --name-only origin/main...HEAD)
+          fi
+          if echo "$DIFF" | grep -q "docker/Dockerfile.datastream-deps"; then
+            echo "Deps Dockerfile changed, building..."
+            docker compose -f docker/docker-compose.yml build datastream-deps
+          else
+            echo "Deps Dockerfile unchanged, pulling from Docker Hub..."
+            docker pull awiciroh/datastream-deps:latest
+          fi
+
+      - name: Build datastream
         run: docker compose -f docker/docker-compose.yml build datastream
 
       - name: Setup and run forcings tests

.github/workflows/test_datastream_python_bmi_conf_generation.yaml

@@ -34,10 +34,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.9
+    - name: Set up Python 3.11
       uses: actions/setup-python@v3
       with:
-        python-version: "3.9"
+        python-version: "3.11"
 
     - name: Install dependencies
       run: |

.github/workflows/test_datastream_python_configuration.yaml

@@ -32,10 +32,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.9
+    - name: Set up Python 3.11
       uses: actions/setup-python@v3
       with:
-        python-version: "3.9"
+        python-version: "3.11"
 
     - name: Install dependencies
       run: |