Local test data

.gitignore

@@ -8,8 +8,12 @@ mock_data/
 simple-eval-pipeline/
 autoeval-jobs/
 .env
+.env.*
+!example.env
 inputs/
 pipeline_log.db
 .vscode/
 viztracer-visualization.json
 pipeline_trace.json
+test/bench-data/pgstac-data/
+.data

README.md

@@ -4,6 +4,18 @@ The coordinator is the entry point to the evaluation pipeline. It takes a gpkg c
 
 The current evaluation pipeline is primarily designed to generate HAND FIM extents or depths and then evaluate these against relevant benchmark sources.
 
+### Getting Started Locally
+1. Create `.env` file
+2. Run `mkdir -p ./.data/nomad/data`
+3. Run `docker compose -f docker-compose-local.yml up`
+4. Register Jobs (see ./local-nomad/README.md)
+5. Load the test stac data by running `./testdata/benchmark/load-test-stac-data.sh`
+6. Create required container images from autoeval-jobs repo
+7. Dispatch a pipeline job through Nomad UI or API
+
+**Tips for working with .env files:**
+- The example.env is a good place to look to make a .env file that is configured for local deployment. This .env file can be stored as .env.local when not in use and copied to .env when local deployment is desired. Depending on which deployment configuration is desired different .env files can be saved locally within the repo without being tracked by git. For example, you could also have a .env.test environment file for deploying to the AWS Test account. 
+
 ### Arguments
 - **HAND Version** 
   - The HAND version argument allows the user to specify a specific version of HAND to generate extents for. This argument is required.

docker-compose.yml → docker-compose-dev.yml

@@ -12,4 +12,4 @@ services:
       - AWS_SECRET_ACCESS_KEY
       - AWS_SESSION_TOKEN
     entrypoint: [ sleep, infinity ]
-    network_mode: "host"
+    network_mode: host

docker-compose-local.yml

@@ -0,0 +1,77 @@
+services:
+
+  stac-fastapi-pgstac:
+    container_name: stac-fastapi-pgstac
+    image: ghcr.io/stac-utils/stac-fastapi-pgstac:4.0.3
+    command: bash -c "python -m stac_fastapi.pgstac.app"
+    healthcheck:
+      test: ["CMD-SHELL", "python -c \"import urllib.request; response = urllib.request.urlopen('http://localhost:8888/_mgmt/ping'); exit(0 if b'PONG' in response.read() else 1)\""]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+      start_period: 30s
+    env_file:
+      - ./.env
+    depends_on:
+      pgstac:
+        condition: service_healthy
+    networks:
+      - autoeval-net
+    ports:
+      - 8888:8888
+
+  pgstac:
+    container_name: pgstac
+    image: ghcr.io/stac-utils/pgstac:v0.9.6
+    volumes:
+      - "./.data/pgstac:/var/lib/postgresql/data"
+    env_file:
+      - ./.env
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U pgstac -d postgis"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    networks:
+      - autoeval-net
+
+
+  asset-server:
+    container_name: asset-server
+    image: nginx:alpine
+    volumes:
+      - ./testdata/benchmark/assets:/usr/share/nginx/html:ro
+    environment:
+      - NGINX_PORT=8889
+    command: |
+      sh -c "sed -i 's/listen       80;/listen       8889;/g' /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;'"
+    networks:
+      - autoeval-net
+    ports:
+      - 8889:8889
+
+  nomad-server:
+    build: ./local-nomad
+    container_name: nomad-server
+    privileged: true
+    volumes:
+      # Docker socket access required for Nomad to manage Docker containers as a task driver
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./local-nomad/nomad-server.hcl:/etc/nomad/nomad.hcl
+      # preserver nomad state
+      # this is only working if .data/nomad/data is created on host beforehand
+      - ./.data/nomad:/nomad
+      # cgroup filesystem access required for Nomad client to manage process isolation and resource limits
+      - /sys/fs/cgroup:/sys/fs/cgroup:rw
+      - /tmp:/tmp
+    environment:
+      - NOMAD_ADDR=${NOMAD_ADDR}
+    command: ["nomad", "agent", "-config", "/etc/nomad/nomad.hcl"]
+    networks:
+      - autoeval-net
+    ports:
+      - 4646:4646
+
+networks:
+  autoeval-net:
+    driver: bridge

example.env

@@ -2,3 +2,32 @@
 AWS_ACCESS_KEY_ID=your_access_key_here
 AWS_SECRET_ACCESS_KEY=your_secret_key_here
 AWS_SESSION_TOKEN=your_session_token_here
+
+# stac-fastapi container environment variables
+POSTGRES_USER=pgstac
+POSTGRES_PASS=password
+POSTGRES_DBNAME=postgis
+POSTGRES_HOST_READER=pgstac
+POSTGRES_HOST_WRITER=pgstac
+APP_HOST=0.0.0.0
+APP_PORT=8888
+RELOAD=true
+ENVIRONMENT=local
+POSTGRES_PORT=5432
+WEB_CONCURRENCY=10
+VSI_CACHE=TRUE
+GDAL_HTTP_MERGE_CONSECUTIVE_RANGES=YES
+GDAL_DISABLE_READDIR_ON_OPEN=EMPTY_DIR
+DB_MIN_CONN_SIZE=1
+DB_MAX_CONN_SIZE=100
+USE_API_HYDRATE=${USE_API_HYDRATE:-false}
+
+# pgstac container environment variables taken from docker compose file in pgstac repo
+POSTGRES_PASSWORD=password
+POSTGRES_DB=postgis
+PGUSER=pgstac
+PGPASSWORD=password
+PGDATABASE=postgis
+
+# Nomad
+NOMAD_ADDR=http://localhost:4646

job_defs/local/agreement_maker.nomad

@@ -1,3 +1,8 @@
+variable "repo_root" {
+  description = "Path to the repository root directory"
+  type        = string
+}
+
 job "agreement_maker" {
   datacenters = ["dc1"] 
   type        = "batch"
@@ -30,13 +35,18 @@ job "agreement_maker" {
       driver = "docker"
 
       config {
-        image = "registry.sh.nextgenwaterprediction.com/ngwpc/fim-c/flows2fim_extents:autoeval-jobs-gval-v0.2" 
-        force_pull = true
-
-        auth {
-          username = "ReadOnly_NGWPC_Group_Deploy_Token" # Or your specific username
-          password = "${NOMAD_META_registry_token}"
-        }
+        # Use local development image - must use specific tag (not 'latest')
+        # to prevent Nomad from trying to pull from a registry
+        image = "autoeval-jobs-gval:local" 
+        force_pull = false
+        network_mode = "host"
+
+        # Mount local test data and output directory
+        volumes = [
+          "${var.repo_root}/testdata:/testdata:ro",
+          "/tmp/autoeval-outputs:/outputs:rw"
+        ]
+
         command = "python3"
         args = [
           "/deploy/agreement_maker/make_agreement.py",
@@ -86,7 +96,8 @@ job "agreement_maker" {
       }
 
       resources {
-        memory = 12000 # Higher memory for large raster processing
+        # set small here for github runner test (8gb total memory)
+        memory = 4000 
       }
 
       logs {

job_defs/local/fim_mosaicker.nomad

@@ -1,3 +1,8 @@
+variable "repo_root" {
+  description = "Path to the repository root directory"
+  type        = string
+}
+
 job "fim_mosaicker" {
   datacenters = ["dc1"] 
   type        = "batch"
@@ -26,13 +31,17 @@ job "fim_mosaicker" {
       driver = "docker"
 
       config {
-        image = "registry.sh.nextgenwaterprediction.com/ngwpc/fim-c/flows2fim_extents:autoeval-jobs-v0.2" 
-        force_pull = true
-
-        auth {
-          username = "ReadOnly_NGWPC_Group_Deploy_Token"
-          password = "${NOMAD_META_registry_token}"
-        }
+        # Use local development image - must use specific tag (not 'latest')
+        # to prevent Nomad from trying to pull from a registry
+        image = "autoeval-jobs:local" 
+        force_pull = false
+        network_mode = "host"
+
+        # Mount local test data and output directory
+        volumes = [
+          "${var.repo_root}/testdata:/testdata:ro",
+          "/tmp/autoeval-outputs:/outputs:rw"
+        ]
 
         command = "python3"
         args = [
@@ -75,7 +84,8 @@ job "fim_mosaicker" {
 
       resources {
         cpu    = 1000 
-        memory = 4096 
+        # set small here for github runner test (8gb total memory)
+        memory = 4000 
       }
 
       logs {

job_defs/local/hand_inundator.nomad

@@ -1,3 +1,8 @@
+variable "repo_root" {
+  description = "Path to the repository root directory"
+  type        = string
+}
+
 job "hand_inundator" {
   datacenters = ["dc1"] 
   type        = "batch"
@@ -28,13 +33,18 @@ job "hand_inundator" {
       driver = "docker"
 
       config {
-        image = "registry.sh.nextgenwaterprediction.com/ngwpc/fim-c/flows2fim_extents:autoeval-jobs-v0.2" 
-        force_pull = true
-
-        auth {
-          username = "ReadOnly_NGWPC_Group_Deploy_Token" # Or your specific username
-          password = "${NOMAD_META_registry_token}"
-        }
+        # Use local development image - must use specific tag (not 'latest')
+        # to prevent Nomad from trying to pull from a registry
+        image = "autoeval-jobs:local" 
+        force_pull = false
+
+        # Mount local test data and output directory
+        volumes = [
+          "${var.repo_root}/testdata:/testdata:ro",
+          "/tmp/autoeval-outputs:/outputs:rw",
+          "/tmp:/tmp:rw"
+        ]
+
         command = "python3"
         args = [
           "/deploy/hand_inundator/inundate.py",
@@ -80,8 +90,8 @@ job "hand_inundator" {
       }
 
       resources {
-        cpu    = 1000 # Adjust CPU MHz (example: 1 core = 1000)
-        memory = 4096 # Adjust Memory MiB (example: 4 GiB)
+        # set small here for github runner test (8gb total memory)
+        memory = 4000
       }
 
       logs {

job_defs/local/pipeline.nomad

@@ -1,3 +1,8 @@
+variable "repo_root" {
+  description = "Path to the repository root directory"
+  type        = string
+}
+
 job "pipeline" {
   datacenters = ["dc1"] 
   type        = "batch"
@@ -29,14 +34,18 @@ job "pipeline" {
       driver = "docker"
 
       config {
-        image = "registry.sh.nextgenwaterprediction.com/ngwpc/fim-c/flows2fim_extents:autoeval-coordinator-v0.1" 
-        force_pull = true
+        # Use local development image - must use specific tag (not 'latest')
+        # to prevent Nomad from trying to pull from a registry
+        image = "autoeval-coordinator:local" 
+        force_pull = false
         network_mode = "host"
 
-        auth {
-          username = "ReadOnly_NGWPC_Group_Deploy_Token"
-          password = "${NOMAD_META_registry_token}"
-        }
+        # Mount local test data and output directory
+        volumes = [
+          "${var.repo_root}/testdata:/testdata:ro",
+          "/tmp/autoeval-outputs:/outputs:rw",
+          "/tmp:/tmp:rw"
+        ]
 
         args = [
           "--aoi", "${NOMAD_META_aoi}",
@@ -58,24 +67,19 @@ job "pipeline" {
 
         # Nomad Configuration
         NOMAD_ADDRESS         = "http://127.0.0.1:4646"
-        NOMAD_TOKEN           = "${NOMAD_TOKEN}"
+        NOMAD_TOKEN           = "${NOMAD_TOKEN}" # this will be changed to a meta variable when the test version of the job is created
         NOMAD_NAMESPACE       = "default"
         NOMAD_REGISTRY_TOKEN  = "${NOMAD_META_registry_token}"
-
-        # S3 Configuration
-        S3_BUCKET             = "fimc-data"
-        S3_BASE_PREFIX        = "${NOMAD_META_output_path}"
-
+
         # Pipeline Configuration
         FIM_TYPE              = "extent"
         HTTP_CONNECTION_LIMIT = "100"
 
         # HAND Index Configuration
-        HAND_INDEX_PARTITIONED_BASE_PATH = "s3://fimc-data/autoeval/hand_output_indices/PI3_uat_and_alpha_domain_3m_wbt/"
         HAND_INDEX_OVERLAP_THRESHOLD_PERCENT = "40.0"
 
         # STAC Configuration
-        STAC_API_URL          = "http://127.0.0.1:8082/"
+        STAC_API_URL          = "http://127.0.0.1:8888/"
         STAC_OVERLAP_THRESHOLD_PERCENT = "40.0"
         STAC_DATETIME_FILTER  = "${NOMAD_META_stac_datetime_filter}"
 
@@ -91,7 +95,7 @@ job "pipeline" {
       }
 
       resources {
-        memory = 8192 
+        memory = 6000 
       }
 
       logs {

local-nomad/README.md

@@ -0,0 +1,14 @@
+After the `nomad-server` is up and running. From the root of the repository execute following command to register jobs. This step is required only one time.
+
+```sh
+docker run --rm --name nomad-job-registrar \
+  -v "$(pwd)/local-nomad/register-jobs.sh:/usr/local/bin/register-jobs.sh" \
+  -v "$(pwd)/job_defs/local:/nomad/jobs" \
+  -v "$(pwd):/repo:ro" \
+  -e "NOMAD_ADDR=http://nomad-server:4646" \
+  -e "REPO_ROOT=$(pwd)" \
+  --network autoeval-coordinator_autoeval-net \
+  --entrypoint /bin/sh \
+  $(docker build -q -f ./local-nomad/Dockerfile ./local-nomad) \
+  -c "/usr/local/bin/register-jobs.sh"
+```