Pre production

.dockerignore

@@ -0,0 +1 @@
+.venv

.env.example

@@ -0,0 +1,9 @@
+DB_DRIVER=postgres
+
+POSTGRES_USER=admin
+POSTGRES_PASSWORD=password
+POSTGRES_DB=
+
+# asset storage
+GCS_BUCKET_NAME=
+GOOGLE_APPLICATION_CREDENTIALS=/path/to/gcs_credentials.json

.gcloudignore

@@ -2,6 +2,17 @@
 # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
 # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
 
+
+tests
+migration
+gcs_credentials.json
+launcher.sh
+test.sh
+docker/*
+alembic/*
+.gitattributes
+.git/*
+
 # User-specific stuff
 .idea/**/workspace.xml
 .idea/**/tasks.xml
@@ -210,6 +221,7 @@ celerybeat.pid
 .venv
 env/
 venv/
+.venv/*
 ENV/
 env.bak/
 venv.bak/

.gitattributes

@@ -0,0 +1 @@
+*.sh text eol=lf

.github/workflows/dev_deploy.yml

@@ -0,0 +1,83 @@
+name: CD (Development)
+
+on:
+  push:
+    branches: [dev]
+
+permissions:
+  contents: write
+
+jobs:
+    staging-deploy:
+
+        runs-on: ubuntu-latest
+        environment: staging
+
+        steps:
+        - name: Check out source repository
+          uses: actions/checkout@v4
+          with:
+            fetch-depth: 0
+
+        - name: Install uv in container
+          uses: astral-sh/setup-uv@v6
+          with:
+            version: "latest"
+
+        - name: Generate requirements.txt
+          run: |
+            uv export -o requirements.txt
+
+        - name: Authenticate to Google Cloud
+          uses: 'google-github-actions/auth@v2'
+          with:
+            credentials_json: ${{ secrets.CLOUD_DEPLOY_SERVICE_ACCOUNT_KEY }}
+
+        # Uses Google Cloud Secret Manager to store secret credentials
+        - name: Create app.yaml
+          run: |
+            echo "service: dev-ocotillo-api" > app.yaml
+            echo "runtime: python313" >> app.yaml
+            echo "entrypoint: gunicorn -w 4 -k uvicorn.workers.UvicornWorker main:app" >> app.yaml
+            echo "instance_class: F4" >> app.yaml
+            echo "inbound_services:" >> app.yaml
+            echo "  - warmup" >> app.yaml
+            echo "automatic_scaling:" >> app.yaml
+            echo "  min_instances: 0" >> app.yaml
+            echo "  max_instances: 10" >> app.yaml
+            echo "" >> app.yaml
+            echo "env_variables:" >> app.yaml
+            echo "  MODE: \"production\"" >> app.yaml
+            echo "  DB_DRIVER: \"cloudsql\"" >> app.yaml
+            echo "  CLOUD_SQL_INSTANCE_NAME: \"${{ secrets.CLOUD_SQL_INSTANCE_NAME }}\"" >> app.yaml
+            echo "  CLOUD_SQL_DATABASE: \"${{ secrets.CLOUD_SQL_DATABASE }}\"" >> app.yaml
+            echo "  CLOUD_SQL_USER: \"${{ secrets.CLOUD_SQL_USER }}\"" >> app.yaml
+            echo "  CLOUD_SQL_PASSWORD: \"${{ secrets.CLOUD_SQL_PASSWORD }}\"" >> app.yaml
+            echo "  GCS_SERVICE_ACCOUNT_KEY: \"${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}\"" >> app.yaml
+            echo "  GCS_BUCKET_NAME: \"${{secrets.GCS_BUCKET_NAME}}\"" >> app.yaml
+            echo "  AUTHENTIK_URL: \"${{secrets.AUTHENTIK_URL}}\"" >> app.yaml
+            echo "  AUTHENTIK_CLIENT_ID: \"${{secrets.AUTHENTIK_CLIENT_ID}}\"" >> app.yaml
+            echo "  AUTHENTIK_AUTHORIZE_URL: \"${{secrets.AUTHENTIK_AUTHORIZE_URL}}\"" >> app.yaml
+            echo "  AUTHENTIK_TOKEN_URL: \"${{secrets.AUTHENTIK_TOKEN_URL}}\"" >> app.yaml
+
+
+        - name: Deploy to Google Cloud
+          run: |
+            gcloud app deploy app.yaml --quiet --project ${{ secrets.GCP_PROJECT_ID }}
+
+        # Clean up old versions - delete only the oldest version, one created and one destroyed
+        - name: Clean up oldest version
+          run: |
+            OLDEST_VERSION=$(gcloud app versions list --service=dev-ocotillo-api --project=${{ secrets.GCP_PROJECT_ID}} --format="value(id)" --sort-by="version.createTime" | head -n 1)
+            if [ ! -z "$OLDEST_VERSION" ]; then
+              echo "Deleting oldest version: $OLDEST_VERSION"
+              gcloud app versions delete $OLDEST_VERSION --service=dev-ocotillo-api --project=${{ secrets.GCP_PROJECT_ID }} --quiet
+              echo "Deleted oldest version: $OLDEST_VERSION"
+            else
+              echo "No versions to delete"
+            fi
+
+        - name: Remove app.yaml
+          run: |
+            rm app.yaml
+

.github/workflows/staging_deploy.yml

@@ -0,0 +1,94 @@
+name: CD (Staging)
+
+on:
+  push:
+    branches: [pre-production]
+
+permissions:
+  contents: write
+
+jobs:
+    staging-deploy:
+
+        runs-on: ubuntu-latest
+        environment: staging
+
+        steps:
+        - name: Check out source repository
+          uses: actions/checkout@v4
+          with:
+            fetch-depth: 0
+
+        - name: Install uv in container
+          uses: astral-sh/setup-uv@v6
+          with:
+            version: "latest"
+
+        - name: Generate requirements.txt
+          run: |
+            uv export -o requirements.txt
+
+        - name: Authenticate to Google Cloud
+          uses: 'google-github-actions/auth@v2'
+          with:
+            credentials_json: ${{ secrets.CLOUD_DEPLOY_SERVICE_ACCOUNT_KEY }}
+
+        # Uses Google Cloud Secret Manager to store secret credentials
+        - name: Create app.yaml
+          run: |
+            echo "service: ocotillo-api-staging" > app.yaml
+            echo "runtime: python313" >> app.yaml
+            echo "entrypoint: gunicorn -w 4 -k uvicorn.workers.UvicornWorker main:app" >> app.yaml
+            echo "instance_class: F4" >> app.yaml
+            echo "inbound_services:" >> app.yaml
+            echo "  - warmup" >> app.yaml
+            echo "automatic_scaling:" >> app.yaml
+            echo "  min_instances: 0" >> app.yaml
+            echo "  max_instances: 10" >> app.yaml
+            echo "" >> app.yaml
+            echo "env_variables:" >> app.yaml
+            echo "  MODE: \"production\"" >> app.yaml
+            echo "  DB_DRIVER: \"cloudsql\"" >> app.yaml
+            echo "  CLOUD_SQL_INSTANCE_NAME: \"${{ secrets.CLOUD_SQL_INSTANCE_NAME }}\"" >> app.yaml
+            echo "  CLOUD_SQL_DATABASE: \"${{ secrets.CLOUD_SQL_DATABASE }}\"" >> app.yaml
+            echo "  CLOUD_SQL_USER: \"${{ secrets.CLOUD_SQL_USER }}\"" >> app.yaml
+            echo "  CLOUD_SQL_PASSWORD: \"${{ secrets.CLOUD_SQL_PASSWORD }}\"" >> app.yaml
+            echo "  GCS_SERVICE_ACCOUNT_KEY: \"${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}\"" >> app.yaml
+            echo "  GCS_BUCKET_NAME: \"${{secrets.GCS_BUCKET_NAME}}\"" >> app.yaml
+            echo "  AUTHENTIK_URL: \"${{secrets.AUTHENTIK_URL}}\"" >> app.yaml
+            echo "  AUTHENTIK_CLIENT_ID: \"${{secrets.AUTHENTIK_CLIENT_ID}}\"" >> app.yaml
+            echo "  AUTHENTIK_AUTHORIZE_URL: \"${{secrets.AUTHENTIK_AUTHORIZE_URL}}\"" >> app.yaml
+            echo "  AUTHENTIK_TOKEN_URL: \"${{secrets.AUTHENTIK_TOKEN_URL}}\"" >> app.yaml
+
+
+        - name: Deploy to Google Cloud
+          run: |
+            gcloud app deploy app.yaml --quiet --project ${{ secrets.GCP_PROJECT_ID }}
+
+        # Clean up old versions - delete only the oldest version, one created and one destroyed
+        - name: Clean up oldest version
+          run: |
+            OLDEST_VERSION=$(gcloud app versions list --service=ocotillo-api-staging --project=${{ secrets.GCP_PROJECT_ID}} --format="value(id)" --sort-by="version.createTime" | head -n 1)
+            if [ ! -z "$OLDEST_VERSION" ]; then
+              echo "Deleting oldest version: $OLDEST_VERSION"
+              gcloud app versions delete $OLDEST_VERSION --service=ocotillo-api-staging --project=${{ secrets.GCP_PROJECT_ID }} --quiet
+              echo "Deleted oldest version: $OLDEST_VERSION"
+            else
+              echo "No versions to delete"
+            fi
+
+        - name: Remove app.yaml
+          run: |
+            rm app.yaml
+
+        # Use PR author's username as git user name
+        - name: Set up git user
+          run: |
+            git config --global user.name "${{ github.actor }}"
+            git config --global user.email "${{ github.actor }}@users.noreply.github.com"
+
+        # ":" are not alloed in git tags, so replace with "-"
+        - name: Tag commit
+          run: |
+            git tag -a "staging-deploy-$(date -u +%Y-%m-%d)T$(date -u +%H-%M-%S%z)" -m "staging gcloud deployment: $(date -u +%Y-%m-%d)T$(date -u +%H:%M:%S%z)"
+            git push origin --tags

.github/workflows/tests.yml

@@ -5,20 +5,20 @@ name: Tests
 
 on:
   pull_request:
-    branches: [ "main",'pre-production']
+    branches: [ "main",'pre-production', 'transfer']
 
 permissions:
   contents: read
 
 jobs:
-  build:
+  run-tests:
     runs-on: ubuntu-latest
 
     services:
-      postgis_timescaledb:
-        image: ghcr.io/dataintegrationgroup/nmdms:latest
+      postgis:
+#        image: ghcr.io/dataintegrationgroup/nmdms:latest
 #        image: postgres
-#        image: postgis/postgis:latest
+        image: postgis/postgis:latest
 #        image: postgis/postgis:17-3.5
 #        image: timescale/timescaledb:2.18.0-pg17
         env:
@@ -63,15 +63,15 @@ jobs:
         DB_DRIVER: postgres
 #        SPATIALITE_LIBRARY_PATH: /usr/lib/x86_64-linux-gnu/mod_spatialite.so
 
-      run: uv run pytest -vv --durations=20 --junitxml=junit.xml
+      run: uv run pytest -vv --durations=20 --cov --cov-report=xml --junitxml=junit.xml
 
-#    - name: Upload results to Codecov
-#      uses: codecov/codecov-action@v4
-#      with:
-#        token: ${{ secrets.CODECOV_TOKEN }}
-#
-#    - name: Upload test results to Codecov
-#      if: ${{ !cancelled() }}
-#      uses: codecov/test-results-action@v1
-#      with:
-#        token: ${{ secrets.CODECOV_TOKEN }}
+    - name: Upload results to Codecov
+      uses: codecov/codecov-action@v4
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+
+    - name: Upload test results to Codecov
+      if: ${{ !cancelled() }}
+      uses: codecov/test-results-action@v1
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -8,7 +8,7 @@ wheels/
 
 # Virtual environments
 .venv
-
+requirements.txt
 
 # JetBrains IDEs
 .idea/
@@ -20,8 +20,11 @@ development.db
 .env
 reset_db.sh
 tests/uploads
-
+migrate.sh
+launcher.sh
+gcs_credentials.json
+transfers/data/assets*
+transfers/transfer*.log
 
 # deployment files
-app.yaml
-alembic.ini
+app.yaml

.pre-commit-config.yaml

@@ -0,0 +1,22 @@
+repos:
+  - repo: https://github.com/psf/black
+    rev: 24.4.2
+    hooks:
+      - id: black
+
+  - repo: https://github.com/pycqa/flake8
+    rev: 7.0.0
+    hooks:
+      - id: flake8
+        args: [
+          '--count',
+          '--select=E9,F63,F7,F82,F401,F541,F631,F634,F701,F702',
+          '--show-source',
+          '--statistics'
+          ]
+        exclude: ^db/__init__.py$ # all models need to be imported for Alembic, but are not used directly
+
+  # - repo: https://github.com/pre-commit/mirrors-mypy
+  #   rev: v1.10.0  # Use the latest stable version or pin to your preference
+  #   hooks:
+  #     - id: mypy

Procfile

@@ -0,0 +1 @@
+web: gunicorn -b :8080 transfers.entrypoint:app -k uvicorn.workers.UvicornWorker