docs(audits): performance, security, and quality audit results

CHANGELOG.md

@@ -6,6 +6,21 @@ All notable changes to the RAJA project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.3.2] - 2026-03-23
+
+### Added
+
+- **`/_perf/` no-auth Envoy route**: Dedicated route to the perf test bucket that bypasses `jwt_authn` and Lua filters entirely, giving a clean direct-S3 baseline for future benchmarks without infrastructure toggling.
+- **`scripts/verify_perf_access.py`**: Pre-flight checker that validates direct S3 access, token issuance, authenticated Envoy GET, and ECS exec connectivity before running a benchmark.
+
+### Fixed
+
+- **Envoy header trust boundary**: Added `request_headers_to_remove: [x-raja-jwt-payload]` to the virtual host in `infra/envoy/envoy.yaml.tmpl`. Clients could previously supply a forged payload header; Lua now only ever sees the value written by `jwt_authn` after successful verification.
+- **Lambda handler type coverage**: Lambda handler directories are now proper packages; `mypy --strict` runs clean across both `src/raja` and `lambda_handlers`.
+- **Deny response metadata leak**: `rale_authorizer` no longer exposes `manifest_hash`, `package_name`, or `registry` in denied authorization responses.
+- **Hard-coded `/tmp` in Lambda**: Replaced with `tempfile.gettempdir()`; `bandit -ll` reports no medium/high findings.
+- **Dependency lockfile drift**: Updated `uv.lock` — `fastapi`, `starlette`, `mangum`, `boto3`, `ruff`, `pydantic-core` brought to current releases.
+
 ## [1.3.1] - 2026-03-19
 
 ### Changed

infra/envoy/entrypoint.sh

@@ -7,6 +7,7 @@ AUTH_DISABLED_VALUE="$(printf '%s' "$AUTH_DISABLED_VALUE" | tr '[:upper:]' '[:lo
 JWKS_ENDPOINT_VALUE="${JWKS_ENDPOINT:-http://localhost:8001/.well-known/jwks.json}"
 RAJA_ISSUER_VALUE="${RAJA_ISSUER:-http://localhost:8000}"
 PUBLIC_PATH_PREFIXES_VALUE="${RAJEE_PUBLIC_PATH_PREFIXES:-}"
+PERF_DIRECT_BUCKET_VALUE="${PERF_DIRECT_BUCKET:-}"
 RALE_AUTHORIZER_URL_VALUE="${RALE_AUTHORIZER_URL:-}"
 RALE_ROUTER_URL_VALUE="${RALE_ROUTER_URL:-}"
 AWS_REGION_VALUE="${AWS_REGION:-${AWS_DEFAULT_REGION:-us-east-1}}"
@@ -166,6 +167,7 @@ fi
 if [ "$AUTH_DISABLED_VALUE" = "1" ] || [ "$AUTH_DISABLED_VALUE" = "true" ] || [ "$AUTH_DISABLED_VALUE" = "yes" ] || [ "$AUTH_DISABLED_VALUE" = "on" ]; then
   AUTH_FILTER=""
   JWT_DEFAULT_RULE=""
+  AUTH_LUA=""
 else
   PUBLIC_PATH_RULES=""
   if [ -n "$PUBLIC_PATH_PREFIXES_VALUE" ]; then
@@ -220,6 +222,65 @@ fi
 
 AUTH_LUA=$(sed 's/^/                          /' /etc/envoy/authorize.lua)
 
+PERF_DIRECT_ROUTE=""
+PERF_CLUSTER=""
+if [ -n "$PERF_DIRECT_BUCKET_VALUE" ]; then
+  PERF_DIRECT_ROUTE=$(cat <<EOF
+                        - match:
+                            prefix: "/_perf/"
+                          route:
+                            cluster: s3_perf_upstream
+                            timeout: 300s
+                            host_rewrite_literal: s3.${AWS_REGION_VALUE}.amazonaws.com
+                            prefix_rewrite: "/"
+                          typed_per_filter_config:
+                            envoy.filters.http.jwt_authn:
+                              "@type": type.googleapis.com/envoy.extensions.filters.http.jwt_authn.v3.PerRouteConfig
+                              disabled: true
+                            envoy.filters.http.lua:
+                              "@type": type.googleapis.com/envoy.extensions.filters.http.lua.v3.LuaPerRoute
+                              disabled: true
+EOF
+)
+  PERF_CLUSTER=$(cat <<EOF
+    - name: s3_perf_upstream
+      type: LOGICAL_DNS
+      dns_lookup_family: V4_ONLY
+      connect_timeout: 5s
+      lb_policy: ROUND_ROBIN
+      typed_extension_protocol_options:
+        envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
+          "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
+          explicit_http_config:
+            http_protocol_options: {}
+          http_filters:
+            - name: envoy.filters.http.aws_request_signing
+              typed_config:
+                "@type": type.googleapis.com/envoy.extensions.filters.http.aws_request_signing.v3.AwsRequestSigning
+                service_name: s3
+                region: ${AWS_REGION_VALUE}
+                use_unsigned_payload: true
+            - name: envoy.filters.http.upstream_codec
+              typed_config:
+                "@type": type.googleapis.com/envoy.extensions.filters.http.upstream_codec.v3.UpstreamCodec
+      load_assignment:
+        cluster_name: s3_perf_upstream
+        endpoints:
+          - lb_endpoints:
+              - endpoint:
+                  address:
+                    socket_address:
+                      address: s3.${AWS_REGION_VALUE}.amazonaws.com
+                      port_value: 443
+      transport_socket:
+        name: envoy.transport_sockets.tls
+        typed_config:
+          "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
+          sni: s3.${AWS_REGION_VALUE}.amazonaws.com
+EOF
+)
+fi
+
 awk -v auth_filter="$AUTH_FILTER" \
     -v auth_lua="$AUTH_LUA" \
     -v public_path_rules="$PUBLIC_PATH_RULES" \
@@ -230,6 +291,8 @@ awk -v auth_filter="$AUTH_FILTER" \
     -v jwks_endpoint="$JWKS_ENDPOINT_VALUE" \
     -v raja_issuer="$RAJA_ISSUER_VALUE" \
     -v jwt_default_rule="$JWT_DEFAULT_RULE" \
+    -v perf_direct_route="$PERF_DIRECT_ROUTE" \
+    -v perf_cluster="$PERF_CLUSTER" \
     '{
       gsub(/__AUTH_FILTER__/, auth_filter)
       gsub(/__PUBLIC_PATH_RULES__/, public_path_rules)
@@ -241,6 +304,8 @@ awk -v auth_filter="$AUTH_FILTER" \
       gsub(/__RALE_CLUSTERS__/, rale_clusters)
       gsub(/__JWKS_ENDPOINT__/, jwks_endpoint)
       gsub(/__RAJA_ISSUER__/, raja_issuer)
+      gsub(/__PERF_DIRECT_ROUTE__/, perf_direct_route)
+      gsub(/__PERF_CLUSTER__/, perf_cluster)
     }1' /etc/envoy/envoy.yaml.tmpl > /tmp/envoy.yaml
 
 if [ "${ENVOY_VALIDATE:-}" = "true" ] || [ "${ENVOY_VALIDATE:-}" = "1" ]; then

infra/envoy/envoy.yaml.tmpl

@@ -25,7 +25,10 @@ static_resources:
                     - name: s3_proxy
                       domains:
                         - "*"
+                      request_headers_to_remove:
+                        - "x-raja-jwt-payload"
                       routes:
+__PERF_DIRECT_ROUTE__
                         - match:
                             prefix: "/"
                           route:
@@ -84,6 +87,7 @@ __AUTH_FILTER__
                       port_value: __JWKS_PORT__
 __JWKS_TRANSPORT_SOCKET__
 __RALE_CLUSTERS__
+__PERF_CLUSTER__
 
 admin:
   address:

infra/terraform/main.tf

@@ -51,7 +51,7 @@ locals {
 
   azs                      = slice(data.aws_availability_zones.available.names, 0, 2)
   rajee_endpoint_protocol  = var.certificate_arn == "" ? "http" : "https"
-  rajee_public_path_prefix = "/${aws_s3_bucket.rajee_test.bucket}"
+  rajee_public_path_prefix = var.perf_test_bucket != "" ? "/${aws_s3_bucket.rajee_test.bucket},/${var.perf_test_bucket}" : "/${aws_s3_bucket.rajee_test.bucket}"
 
   rajee_public_grants = var.use_public_grants ? [
     "s3:GetObject/${aws_s3_bucket.rajee_test.bucket}/rajee-integration/",
@@ -1254,6 +1254,56 @@ resource "aws_iam_role_policy" "rale_router_permissions" {
   })
 }
 
+resource "aws_iam_role_policy" "rale_router_perf_bucket" {
+  count = var.perf_test_bucket != "" ? 1 : 0
+  name  = "${var.stack_name}-rale-router-perf-bucket"
+  role  = aws_iam_role.rale_router_lambda.id
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Sid    = "PerfTestBucketRead"
+        Effect = "Allow"
+        Action = [
+          "s3:GetObject",
+          "s3:GetObjectVersion",
+          "s3:ListBucket"
+        ]
+        Resource = [
+          "arn:aws:s3:::${var.perf_test_bucket}",
+          "arn:aws:s3:::${var.perf_test_bucket}/*"
+        ]
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role_policy" "rajee_task_perf_bucket" {
+  count = var.perf_test_bucket != "" ? 1 : 0
+  name  = "${var.stack_name}-rajee-task-perf-bucket"
+  role  = aws_iam_role.rajee_task.id
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Sid    = "PerfDirectBucketRead"
+        Effect = "Allow"
+        Action = [
+          "s3:GetObject",
+          "s3:GetObjectVersion",
+          "s3:ListBucket"
+        ]
+        Resource = [
+          "arn:aws:s3:::${var.perf_test_bucket}",
+          "arn:aws:s3:::${var.perf_test_bucket}/*"
+        ]
+      }
+    ]
+  })
+}
+
 resource "aws_lambda_function" "rale_router" {
   function_name = "${var.stack_name}-rale-router"
   role          = aws_iam_role.rale_router_lambda.arn
@@ -1736,6 +1786,17 @@ resource "aws_iam_role_policy" "rajee_task_permissions" {
           local.rale_authorizer_lambda_arn,
           local.rale_router_lambda_arn
         ]
+      },
+      {
+        Sid    = "ECSExec"
+        Effect = "Allow"
+        Action = [
+          "ssmmessages:CreateControlChannel",
+          "ssmmessages:CreateDataChannel",
+          "ssmmessages:OpenControlChannel",
+          "ssmmessages:OpenDataChannel"
+        ]
+        Resource = ["*"]
       }
     ]
   })
@@ -1798,6 +1859,10 @@ resource "aws_ecs_task_definition" "rajee" {
           name  = "RAJEE_PUBLIC_PATH_PREFIXES"
           value = local.rajee_public_path_prefix
         },
+        {
+          name  = "PERF_DIRECT_BUCKET"
+          value = var.perf_test_bucket
+        },
         {
           name  = "RAJEE_PUBLIC_GRANTS"
           value = join(",", local.rajee_public_grants)
@@ -1982,6 +2047,7 @@ resource "aws_ecs_service" "rajee" {
   task_definition                    = aws_ecs_task_definition.rajee.arn
   desired_count                      = 2
   launch_type                        = "FARGATE"
+  enable_execute_command             = true
   health_check_grace_period_seconds  = 30
   deployment_minimum_healthy_percent = 50
   deployment_maximum_percent         = 200

infra/terraform/variables.tf

@@ -151,6 +151,12 @@ variable "datazone_guests_project_name" {
   default     = "raja-guests"
 }
 
+variable "perf_test_bucket" {
+  description = "External S3 bucket used for performance benchmarks (e.g. data-yaml-spec-tests). Grants the RALE router Lambda read access and adds the bucket prefix to RAJEE_PUBLIC_PATH_PREFIXES so the auth-disabled Envoy baseline can reach it."
+  type        = string
+  default     = "data-yaml-spec-tests"
+}
+
 variable "datazone_package_asset_type" {
   description = "Custom Amazon DataZone asset type name used for Quilt package listings."
   type        = string

infra/tf-outputs.json

@@ -1 +1 @@
-{"api_url": "https://wezevk884h.execute-api.us-east-1.amazonaws.com/prod/", "control_plane_lambda_arn": "arn:aws:lambda:us-east-1:712023778557:function:raja-standalone-control-plane", "datazone_domain_id": "dzd-6w14ep5r5owwh3", "datazone_guests_environment_role_arn": "arn:aws:iam::712023778557:role/raja-dz-env-guests", "datazone_guests_project_id": "b3byg401pnpjjb", "datazone_owner_environment_role_arn": "arn:aws:iam::712023778557:role/raja-dz-env-owner", "datazone_owner_project_id": "ag00w9am11jcx3", "datazone_package_asset_type": "QuiltPackage", "datazone_package_asset_type_revision": "2", "datazone_portal_url": "https://dzd-6w14ep5r5owwh3.sagemaker.us-east-1.on.aws", "datazone_users_environment_role_arn": "arn:aws:iam::712023778557:role/raja-dz-env-users", "datazone_users_project_id": "bm7eqh5dc6olrb", "ecs_cluster_name": "raja-standalone-rajee-cluster", "ecs_service_name": "raja-standalone-rajee-service", "envoy_image_tag": "4577563f", "envoy_repository_uri": "712023778557.dkr.ecr.us-east-1.amazonaws.com/raja/envoy", "iceberg_lf_database_name": "raja-standalone-iceberg-lf", "jwt_secret_arn": "arn:aws:secretsmanager:us-east-1:712023778557:secret:raja-standalone-jwt-signing-key-UhfCGh", "rajee_admin_url": "", "rajee_endpoint": "http://raja-standalone-rajee-alb-2076392115.us-east-1.elb.amazonaws.com", "rajee_registry_bucket_name": "raja-poc-registry-712023778557-us-east-1", "rajee_test_bucket_name": "raja-poc-test-712023778557-us-east-1", "rale_authorizer_arn": "arn:aws:lambda:us-east-1:712023778557:function:raja-standalone-rale-authorizer", "rale_authorizer_url": "https://pz3444rqvmy6foufbjovcc3phu0lgbnu.lambda-url.us-east-1.on.aws/", "rale_router_arn": "arn:aws:lambda:us-east-1:712023778557:function:raja-standalone-rale-router", "rale_router_url": "https://ueiilftoaxf3fmv5b25iuike240plpvp.lambda-url.us-east-1.on.aws/", "datazone_projects": "{\"alpha\": {\"project_id\": \"ag00w9am11jcx3\", \"project_label\": \"Alpha\", \"environment_id\": \"dmko3lxbkmo4yv\"}, \"bio\": {\"project_id\": \"bm7eqh5dc6olrb\", \"project_label\": \"Bio\", \"environment_id\": \"ax3bgrnehcwfxj\"}, \"compute\": {\"project_id\": \"b3byg401pnpjjb\", \"project_label\": \"Compute\", \"environment_id\": \"4cr6qj7biuk5d3\"}}", "datazone_project_ids": {"alpha": "ag00w9am11jcx3", "bio": "bm7eqh5dc6olrb", "compute": "b3byg401pnpjjb"}, "datazone_project_environment_ids": {"alpha": "dmko3lxbkmo4yv", "bio": "ax3bgrnehcwfxj", "compute": "4cr6qj7biuk5d3"}}
+{"api_url": "https://wezevk884h.execute-api.us-east-1.amazonaws.com/prod/", "control_plane_lambda_arn": "arn:aws:lambda:us-east-1:712023778557:function:raja-standalone-control-plane", "datazone_domain_id": "dzd-6w14ep5r5owwh3", "datazone_guests_environment_role_arn": "arn:aws:iam::712023778557:role/raja-dz-env-guests", "datazone_guests_project_id": "b3byg401pnpjjb", "datazone_owner_environment_role_arn": "arn:aws:iam::712023778557:role/raja-dz-env-owner", "datazone_owner_project_id": "ag00w9am11jcx3", "datazone_package_asset_type": "QuiltPackage", "datazone_package_asset_type_revision": "2", "datazone_portal_url": "https://dzd-6w14ep5r5owwh3.sagemaker.us-east-1.on.aws", "datazone_users_environment_role_arn": "arn:aws:iam::712023778557:role/raja-dz-env-users", "datazone_users_project_id": "bm7eqh5dc6olrb", "ecs_cluster_name": "raja-standalone-rajee-cluster", "ecs_service_name": "raja-standalone-rajee-service", "envoy_image_tag": "4bd66c16", "envoy_repository_uri": "712023778557.dkr.ecr.us-east-1.amazonaws.com/raja/envoy", "iceberg_lf_database_name": "raja-standalone-iceberg-lf", "jwt_secret_arn": "arn:aws:secretsmanager:us-east-1:712023778557:secret:raja-standalone-jwt-signing-key-UhfCGh", "rajee_admin_url": "", "rajee_endpoint": "http://raja-standalone-rajee-alb-2076392115.us-east-1.elb.amazonaws.com", "rajee_registry_bucket_name": "raja-poc-registry-712023778557-us-east-1", "rajee_test_bucket_name": "raja-poc-test-712023778557-us-east-1", "rale_authorizer_arn": "arn:aws:lambda:us-east-1:712023778557:function:raja-standalone-rale-authorizer", "rale_authorizer_url": "https://pz3444rqvmy6foufbjovcc3phu0lgbnu.lambda-url.us-east-1.on.aws/", "rale_router_arn": "arn:aws:lambda:us-east-1:712023778557:function:raja-standalone-rale-router", "rale_router_url": "https://ueiilftoaxf3fmv5b25iuike240plpvp.lambda-url.us-east-1.on.aws/", "datazone_projects": "{\"alpha\": {\"project_id\": \"ag00w9am11jcx3\", \"project_label\": \"Alpha\", \"environment_id\": \"dmko3lxbkmo4yv\"}, \"bio\": {\"project_id\": \"bm7eqh5dc6olrb\", \"project_label\": \"Bio\", \"environment_id\": \"ax3bgrnehcwfxj\"}, \"compute\": {\"project_id\": \"b3byg401pnpjjb\", \"project_label\": \"Compute\", \"environment_id\": \"4cr6qj7biuk5d3\"}}", "datazone_project_ids": {"alpha": "ag00w9am11jcx3", "bio": "bm7eqh5dc6olrb", "compute": "b3byg401pnpjjb"}, "datazone_project_environment_ids": {"alpha": "dmko3lxbkmo4yv", "bio": "ax3bgrnehcwfxj", "compute": "4cr6qj7biuk5d3"}}

lambda_handlers/control_plane/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations

lambda_handlers/control_plane/handler.py

@@ -4,4 +4,6 @@
 
 from raja.server.app import app
 
+__all__ = ["handler"]
+
 handler = Mangum(app)

lambda_handlers/package_resolver/handler.py

@@ -1,14 +1,18 @@
 from __future__ import annotations
 
 from raja.manifest import package_membership_checker, resolve_package_manifest, resolve_package_map
+from raja.models import S3Location
+from raja.package_map import PackageMap
 
+__all__ = ["resolve_manifest", "resolve_translation_map", "check_membership"]
 
-def resolve_manifest(quilt_uri: str):
+
+def resolve_manifest(quilt_uri: str) -> list[S3Location]:
     """Resolve a Quilt+ URI to a list of physical locations."""
     return resolve_package_manifest(quilt_uri)
 
 
-def resolve_translation_map(quilt_uri: str):
+def resolve_translation_map(quilt_uri: str) -> PackageMap:
     """Resolve a Quilt+ URI to a logical-to-physical map."""
     return resolve_package_map(quilt_uri)
 

lambda_handlers/rale_authorizer/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations

lambda_handlers/rale_authorizer/handler.py

@@ -2,6 +2,7 @@
 
 import json
 import os
+import tempfile
 from typing import Any
 
 import boto3
@@ -11,6 +12,8 @@
 from raja.quilt_uri import parse_quilt_uri
 from raja.token import create_taj_token
 
+__all__ = ["handler"]
+
 
 def _response(status_code: int, body: dict[str, Any]) -> dict[str, Any]:
     return {
@@ -77,8 +80,18 @@ def _extract_principal(event: dict[str, Any]) -> str:
 
     payload_raw = headers.get("x-raja-jwt-payload")
     if payload_raw and trusted_forwarder:
+        import base64
+
+        payload_str = payload_raw
+        if not payload_str.startswith("{"):
+            # Envoy forwards the JWT payload as base64url; decode it first.
+            padded = payload_str + "=" * (-len(payload_str) % 4)
+            try:
+                payload_str = base64.urlsafe_b64decode(padded).decode("utf-8")
+            except Exception:
+                payload_str = payload_raw
         try:
-            payload = json.loads(payload_raw)
+            payload = json.loads(payload_str)
         except json.JSONDecodeError:
             payload = {}
         subject = payload.get("sub")
@@ -149,11 +162,12 @@ def _parse_usl(raw_path: str) -> tuple[str, str, str | None, str, str | None]:
 
 
 def _resolve_latest_hash_via_quilt3(registry: str, package_name: str) -> str:
-    os.environ.setdefault("HOME", "/tmp")
-    os.environ.setdefault("XDG_DATA_HOME", "/tmp")
-    os.environ.setdefault("XDG_CACHE_HOME", "/tmp")
+    temp_dir = tempfile.gettempdir()
+    os.environ.setdefault("HOME", temp_dir)
+    os.environ.setdefault("XDG_DATA_HOME", temp_dir)
+    os.environ.setdefault("XDG_CACHE_HOME", temp_dir)
     try:
-        import quilt3  # type: ignore[import-not-found]
+        import quilt3  # type: ignore[import-untyped]
     except Exception as exc:  # pragma: no cover - exercised via integration tests
         raise RuntimeError("quilt3 is required for package resolution") from exc
     package = quilt3.Package.browse(
@@ -222,27 +236,19 @@ def handler(event: dict[str, Any], context: Any) -> dict[str, Any]:  # noqa: ARG
         allowed = service.has_package_grant(project_id=project_id, quilt_uri=quilt_uri)
     except DataZoneError:
         return _response(503, {"error": "authorization service unavailable"})
-    except (ClientError, BotoCoreError):
+    except ClientError, BotoCoreError:
         return _response(503, {"error": "authorization service unavailable"})
 
     if not allowed:
-        return _response(
-            403,
-            {
-                "decision": "DENY",
-                "manifest_hash": manifest_hash,
-                "package_name": package_name,
-                "registry": registry,
-            },
-        )
+        return _response(403, {"decision": "DENY"})
 
     secrets = boto3.client("secretsmanager", region_name=region)
     try:
         secret_kwargs: dict[str, str] = {"SecretId": jwt_secret_arn}
         if jwt_secret_version:
             secret_kwargs["VersionId"] = jwt_secret_version
         jwt_secret = secrets.get_secret_value(**secret_kwargs)["SecretString"]
-    except (ClientError, BotoCoreError, KeyError):
+    except ClientError, BotoCoreError, KeyError:
         return _response(503, {"error": "failed to load jwt secret"})
 
     token_ttl = int(os.environ.get("TOKEN_TTL", "3600"))

lambda_handlers/rale_router/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations