From cd25645cf1b8a6aeff718427ceb897976ebd7898 Mon Sep 17 00:00:00 2001 From: shapirov103 Date: Mon, 13 Jul 2026 17:46:03 -0400 Subject: [PATCH 1/4] docs(architecture): identity + token-exchange decisions and roadmap ADRs for secretless workload identity (Shape A gateway identity; OAM identity traits; aws-service-identity/XPodIdentity/env-config; per-cluster eks_oidc_provider) and the user-delegated token-exchange roadmap, incl. the pending agentgateway release that ships backend.auth.oauthTokenExchange (v1.1.0 lacks it; PRs #2189/#2458). --- .../agent-identity-and-token-exchange.md | 233 ++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100644 docs/architecture/agent-identity-and-token-exchange.md diff --git a/docs/architecture/agent-identity-and-token-exchange.md b/docs/architecture/agent-identity-and-token-exchange.md new file mode 100644 index 00000000..3f5ee8c5 --- /dev/null +++ b/docs/architecture/agent-identity-and-token-exchange.md @@ -0,0 +1,233 @@ +# Agent Identity & Token Exchange — Architecture Decisions + +Status: living document. Captures the significant decisions behind the secretless +workload-identity model and the roadmap to user-delegated (on-behalf-of) access. + +Repos / branches: +- OAP (this repo): `aws-samples/sample-open-agentic-platform` — branch `feature/oam-for-agents` (PR #33 → `main`). +- Platform: `aws-samples/appmod-blueprints` — branch `feature/agent-platform-shapirov`. + +Reference environment: cluster `peeks-hub`, account `929819487611`, `us-west-2`. +EKS OIDC issuer: `https://oidc.eks.us-west-2.amazonaws.com/id/1BABC5C7BFD3BFE9636A486678E1D6F6`. +Component versions: agentgateway `v1.1.0`, Crossplane `v2.2.1` +(functions: environment-configs v0.3.0, patch-and-transform v0.10.0, cel-filter v0.2.0), +Keycloak `26.3.3`, Bifrost `2.1.16`, vela CLI `1.10.7`. + +--- + +## Goal + +Every agent gets a **secretless identity**. Two trust domains: +1. **AgentGateway / MCP** — so agents reach MCP servers and other agents. +2. **AWS** — so agents call AWS APIs (Bedrock, AgentCore memory, etc.). + +North star: when an agent calls MCP / another agent **on behalf of a user**, the +downstream request must carry the **user as subject** and the **agent as actor** +(delegation) — for (a) audit logs that show "API call X executed by agent Y on +behalf of user Z", and (b) authorization decisions based on the invoking +user/group/role, not just the agent's blanket workload trust. + +--- + +## ADR-1 — LLM gateway is Bifrost (context) + +Migrated LiteLLM → Bifrost. Bedrock via EKS Pod Identity; model alias +`claude-sonnet`. Agent uses Strands `OpenAIModel` against Bifrost `/v1`. Governance +VK auth currently disabled (`enforceAuthOnInference: false`). Not identity-critical +but sets the "gateway is the boundary" pattern. + +## ADR-2 — Inbound workload identity = "Shape A" (gateway trusts the cluster EKS OIDC) + +**Decision.** Agents authenticate to AgentGateway with their **projected Kubernetes +ServiceAccount token** (audience `agentgateway`). AgentGateway validates it against +the **cluster's own EKS OIDC issuer**, added as a *second* JWT provider on the +gateway policy (alongside the Keycloak provider used for humans). + +**Alternatives rejected:** +- *Keycloak-issued workload token (inbound "Shape B")* — Keycloak's external-token + grants (JWT Authorization Grant / legacy token exchange) require a **confidential + client secret** and a **per-workload linked Keycloak user**; not secretless, heavy. +- *client_credentials with a per-workload secret* — a secret per workload; rejected. +- *SPIRE* — heavier infra; kept as a fallback if the SA-token path proves insufficient. + +**Rationale.** No secret; kubelet auto-rotates the token; per-workload identity is +free (`sub = system:serviceaccount::`); **environment isolation is free** +(dev/prod clusters have distinct OIDC issuers, so a dev token is cryptographically +invalid at the prod gateway). Keycloak remains the **human** IdP unchanged. + +**Consequences / how.** +- Gateway `AgentgatewayPolicy.traffic.jwtAuthentication.providers` gets a 2nd + provider (issuer = cluster OIDC, JWKS over a static `AgentgatewayBackend` + host `oidc.eks..amazonaws.com:443` with inline `policies.tls: {}`). +- Authz CEL allows both shapes: + `(has(jwt.realm_access) && jwt.realm_access.roles.exists(r, r=="default-roles-platform")) || jwt.sub.startsWith("system:serviceaccount:")`. +- The agent app reads the token from `WORKLOAD_TOKEN_PATH` and sends + `Authorization: Bearer` to MCP (via `streamablehttp_client(headers=...)`). + +**Verified:** agent pod → gateway → `mcp-time` returns tools (HTTP 200; was 401 +before the provider + agent-code wiring). + +## ADR-3 — Identity is composed via OAM traits; the agent owns its ServiceAccount + +**Decision.** Identity is modeled as **KubeVela traits** attached to a component, +not baked into each ComponentDefinition: +- `gateway-identity` — projects the `agentgateway`-audience SA token + sets `WORKLOAD_TOKEN_PATH`. +- `aws-service-identity` — grants AWS IAM identity (see ADR-4). + +**Key constraint that drove the design:** a **trait can only read `context.name`** +(the component name) — it *cannot* read a component's `parameter.name`. So anything +a trait must reference (the ServiceAccount, the container) has to be named +`context.name`. Therefore: +- The `agent` component was refactored to key **everything** off `context.name` / + `context.namespace` (Rollout, Services, container, SA, `AGENT_NAME`, gateway route) + and to **own a dedicated ServiceAccount** = `context.name`. The `name`, + `namespace`, and `serviceAccount` parameters were **removed** (breaking change: + OAM Applications drop `properties.name/namespace`). +- Cloud-agnostic naming: `aws-service-identity` (future `gcp-service-identity`, …). +- Added a generic `service-rollout` component (Argo Rollout + health gating, owns + its SA) as the base for non-agent workloads. `appmod-service`/`dp-service-account` + kept for compatibility. + +## ADR-4 — AWS identity = Pod Identity, "Option C" (self-inject + init-wait) via an XPodIdentity Composition fed by `env-config` + +**Decision.** `aws-service-identity` emits a **`PodIdentity` claim** +(`platform.gitops.io`, appmod XRD/Composition). The **`XPodIdentity` Composition** +resolves `clusterName`/`region` from the ambient **`env-config` EnvironmentConfig** +(via `function-environment-configs`) and creates the IAM Role +(`-role`) + `PodIdentityAssociation`. The developer passes **no +cluster parameters** — only optional `accessFor` (sibling component policies). + +**Why "Option C" (self-inject + init-wait).** EKS Pod Identity injects creds via a +**mutating webhook at pod admission**, which only fires if the association already +exists → a race that broke the pure-trait approach historically. Option C makes the +pod self-inject the creds env (`AWS_CONTAINER_CREDENTIALS_FULL_URI`) + the +`pods.eks.amazonaws.com` projected token, and adds a `wait-for-aws-identity` init +container that blocks until `aws sts get-caller-identity` succeeds. **Verified on +cluster:** the EKS webhook *skips* injection when the creds env is already present +(no duplicate-volume conflict), and STS resolves without a region env. + +**`env-config` is the ambient metadata contract.** A cluster-scoped Crossplane +`EnvironmentConfig` named `env-config` on every cluster, carrying at least +`clusterName`, `region`, `vpcId`, `privateSubnetIds`, `publicSubnetIds`. Only +Compositions can consume it (not KubeVela, not raw MRs) — hence the `XPodIdentity` +Composition indirection. + +**Verified:** `PodIdentity` claim → Role + `PodIdentityAssociation` Ready +(`clusterName=peeks-hub` from `env-config`); agent pod init-wait logs "AWS identity +ready"; in-pod STS returns the assumed role. + +## ADR-5 — Per-cluster EKS OIDC issuer surfaced as the `eks_oidc_provider` annotation + +**Decision.** The gateway's workload JWT provider is templated per-cluster from an +`eks_oidc_provider` cluster-secret annotation: +- **Spokes:** the `platform-cluster` Crossplane Composition writes it from + `status.oidcIssuer` (same Observe+Update `Object` pattern as `aws_vpc_id`); + automatic for every spoke at provision time. +- **Hub:** set by the OAP `Taskfile` (`agentic:hub-oidc-annotation`), because the + hub is bootstrapped from a kind cluster before the platform exists (chicken-and-egg). + +## Key gotchas (do not re-discover) + +- **Crossplane ProviderConfig is `default`**, not `provider-aws-config` (the latter + is stale in `dp-service-account`). Applies to the classic `*.aws.upbound.io` + provider. The namespaced `*.aws.m.upbound.io` family has **no** ProviderConfig on + this cluster — so use the **classic** provider (this is why `agentcore-memory` was + switched from `bedrockagentcore.aws.m.upbound.io` → `bedrockagentcore.aws.upbound.io` + + `default`). +- **Keycloak 26.3.3 token-exchange matrix:** Standard Token Exchange v2 (GA) is + internal-internal only (subject must be a Keycloak access token). External-token → + Keycloak token needs Legacy Token Exchange V1 (preview, deprecated) or JWT + Authorization Grant (26.5 preview) — both require a **confidential client** and a + **linked user**. This is why inbound identity is Shape A, not Keycloak-brokered. +- ArgoCD stuck sync operations pin an old git revision; terminate with + `kubectl patch app -n argocd --type merge -p '{"operation":null}'` then refresh. +- OAM defs must be lowercase-hyphen (RFC-1123). Regenerate with + `bash platform/oam/generate.sh` after editing CUE; commit both `.cue` and generated YAML. + +--- + +# Roadmap — User-delegated access via gateway token exchange + +Target = ADR-6 (below). This is the mechanism that "checks the security boxes": +per-call audit of *user + agent*, and user/group/role-based authorization on MCP/API +calls. + +## ADR-6 (TARGET) — User on-behalf-of via AgentGateway backend token exchange + +**Model.** Gateway-side **backend** auth exchanges the inbound bearer for a +downstream token before calling the upstream, using **RFC 8693 delegation**: +`subject_token = user token`, `actor_token = agent's SA token` → downstream token +with `sub = user`, `act = agent`, audience-scoped to the target MCP/agent. Secrets +(the exchange client credential) live at the **gateway** (a k8s Secret), never in +the agent. Our existing pieces compose: **Shape A SA token = the `actor`**, the +**Keycloak user token = the `subject`**. + +**Standards / grants** (all under agentgateway `backendAuth.oauthTokenExchange`): +- RFC 8693 token exchange (`subject_token`, plus `actorToken`, `resources` per RFC 8707). +- RFC 7523 jwt-bearer / JWT assertion (`assertion`) — matches Keycloak JWT Authorization Grant. +- Entra OBO (jwt-bearer + `requested_token_use=on_behalf_of`). +Multi-hop (agent→agent→MCP) = OAuth Identity & Authorization Chaining / ID-JAG +(token-exchange + jwt-bearer composition; agentgateway `cross_app_access`). + +### Blocker — pending agentgateway release + +- **Feature status in agentgateway:** MERGED to `main`. Data plane in + `crates/agentgateway/src/http/auth/oauth/` (mod/transport/cross_app_access), + controller `backend_policies.go` + `agentgateway_policy_types.go`, e2e tests, and + `examples/traffic-token-exchange/{oauth-rfc8693,jwt-authz-grant}`. PRs **#2189** + (data plane) and **#2458** (controller). Blog: agentgateway.dev/blog/2026-07-12-…-token-exchange-jwt-assertion-entra-obo. +- **Installed version:** agentgateway **v1.1.0** (proxy + controller). Its + `AgentgatewayPolicy` `spec.backend.auth` keys are + `[aws, azure, gcp, key, passthrough, secretRef]` — **no `oauthTokenExchange`**. +- **Therefore:** token exchange is NOT usable on our cluster yet. It requires + upgrading agentgateway to the release that ships #2189/#2458 (post-v1.1.0; as of + 2026-07-13 appears to still be `main`/pre-release — verify a tagged release before + the bump). + +### Phased plan + +1. **Track & upgrade agentgateway.** Wait for / pin the release exposing + `backend.auth.oauthTokenExchange`; bump the `agentgateway`/`agentgateway-crds` + addons; re-verify the CRD has the field. +2. **Propagate user identity through the agent (the main app change).** Today the + agent calls MCP with its *own* SA token. For OBO it must capture the inbound + **user** token (A2A auth passthrough) and forward it as the subject on outbound + MCP calls; the SA token becomes the actor. This is the largest new piece. +3. **Attach a backend exchange policy** (`AgentgatewayPolicy` with + `backend.auth.oauthTokenExchange`) in front of the MCP backends: `subject_token` + = user token, `actorToken` = agent SA token, audience-scoped per tool; token + endpoint as an `AgentgatewayBackend`, client secret from a k8s Secret. +4. **Authorization at the gateway.** Enforce user/group/role (from the user's + claims) as the chokepoint — a tool call is allowed only if the invoking user is + authorized, not merely because the agent workload is trusted. +5. **Multi-hop chaining (ID-JAG)** for agent→agent→MCP, preserving the original user + and accumulating the actor chain so the audit trail stays intact. + +### Open questions to resolve during the spike + +- **Keycloak delegation support:** can our Keycloak issue a genuine RFC 8693 + delegation token with an `act` claim (vs impersonation)? Historically Keycloak is + strongest on internal-internal + impersonation; verify before committing. (May + require Keycloak 26.5 JWT Authorization Grant / config, or gateway-constructed + delegation.) +- **Token lifetime:** user tokens are ~1h; the exchanged token TTL is capped by the + subject `exp`. Long autonomous tasks need a refresh/offline strategy or must be + bounded to the user session. +- **Not fully secretless:** the exchange requires a client secret, but at the + **gateway** (correct place), not per-agent; the inbound SA-token path stays secretless. + +--- + +## Current state (implemented + verified) + +| Area | State | +|---|---| +| Bifrost LLM gateway (Bedrock, claude-sonnet) | done, verified | +| Gateway identity (Shape A) — SA token validated by cluster EKS OIDC | done, verified (agent→gateway→mcp-time 200) | +| `gateway-identity` trait + agent reads `WORKLOAD_TOKEN_PATH` | done, verified | +| `aws-service-identity` trait → `XPodIdentity` → Role + PodIdentityAssociation | done, verified (STS in-pod) | +| `env-config` EnvironmentConfig (clusterName/region) | live on hub | +| `service-rollout` component; `agent` refactored to `context.name` + owns SA | done | +| `agentcore-memory` classic provider + `default` | done, memory provisions | +| Per-cluster `eks_oidc_provider` (spoke Composition; hub Taskfile) | done | +| **User-delegated token exchange (OBO)** | **blocked on agentgateway release (ADR-6)** | From 0880dbf3cc208f65a9a6f1de0877f2d9d2c5b73f Mon Sep 17 00:00:00 2001 From: shapirov103 Date: Mon, 20 Jul 2026 16:58:40 -0400 Subject: [PATCH 2/4] fix(oam): gate aws-service-identity on PodIdentity readiness; add agent resource requests aws-service-identity trait: replace the STS-based init wait with a distroless, digest-pinned Chainguard kubectl 'wait --for=condition=Ready' on the PodIdentity resource, plus a namespaced Role/RoleBinding (get/list/watch podidentities) so the pod ServiceAccount can read it. Control-plane readiness only; the brief IAM data-plane propagation window is handled by app-level retry. example-agent-agentcore-memory: add CPU/memory requests (+ mem limit) so the agent isn't CPU-starved on contended nodes (root cause of the liveness-kill crashloops). Regenerated oam-agent-components templates. --- .../oam-agent-components/templates/agent.yaml | 1 + .../templates/aws-service-identity.yaml | 57 ++++++++++++-- .../traits/aws-service-identity.cue | 75 ++++++++++++++++--- .../example-agent-agentcore-memory.yaml | 9 +++ 4 files changed, 122 insertions(+), 20 deletions(-) diff --git a/gitops/addons/charts/oam-agent-components/templates/agent.yaml b/gitops/addons/charts/oam-agent-components/templates/agent.yaml index 7b73ce91..215a2806 100644 --- a/gitops/addons/charts/oam-agent-components/templates/agent.yaml +++ b/gitops/addons/charts/oam-agent-components/templates/agent.yaml @@ -1,4 +1,5 @@ # Code generated by KubeVela templates. DO NOT EDIT. Please edit the original cue file. +# # Code generated from CUE definitions. DO NOT EDIT. apiVersion: core.oam.dev/v1beta1 kind: ComponentDefinition metadata: diff --git a/gitops/addons/charts/oam-agent-components/templates/aws-service-identity.yaml b/gitops/addons/charts/oam-agent-components/templates/aws-service-identity.yaml index 3aea90db..6f829434 100644 --- a/gitops/addons/charts/oam-agent-components/templates/aws-service-identity.yaml +++ b/gitops/addons/charts/oam-agent-components/templates/aws-service-identity.yaml @@ -21,8 +21,9 @@ spec: accessFor?: [...string] // +usage=Container to inject AWS credentials into (defaults to the component name) containerName: *context.name | string - // +usage=Image for the init container that waits for AWS identity readiness - waitImage: *"public.ecr.aws/aws-cli/aws-cli:latest" | string + // +usage=Distroless kubectl image for the PodIdentity-readiness init gate (entrypoint = kubectl). + // Chainguard kubectl:latest, pinned by multi-arch index digest (amd64+arm64) for immutability. + waitImage: *"public.ecr.aws/chainguard/kubectl:latest@sha256:5cd49041fed950723afaefcd141a163e5a5306f243841510d3e1e3667b0cdfb9" | string } // Self-injected creds URI + token mount (the EKS webhook skips injection when @@ -54,6 +55,40 @@ spec: } } + // RBAC so the pod's ServiceAccount can read its own PodIdentity resource + // (used by the wait-for-pod-identity init container). Namespaced, read-only. + "\(context.name)-podidentity-reader-role": { + apiVersion: "rbac.authorization.k8s.io/v1" + kind: "Role" + metadata: { + name: "\(context.name)-podidentity-reader" + namespace: context.namespace + } + rules: [{ + apiGroups: ["platform.gitops.io"] + resources: ["podidentities"] + verbs: ["get", "list", "watch"] + }] + } + "\(context.name)-podidentity-reader-binding": { + apiVersion: "rbac.authorization.k8s.io/v1" + kind: "RoleBinding" + metadata: { + name: "\(context.name)-podidentity-reader" + namespace: context.namespace + } + roleRef: { + apiGroup: "rbac.authorization.k8s.io" + kind: "Role" + name: "\(context.name)-podidentity-reader" + } + subjects: [{ + kind: "ServiceAccount" + name: context.name + namespace: context.namespace + }] + } + // Attach sibling components' IAM policies to the role the Composition // creates (deterministic name "-role"). if parameter.accessFor != _|_ { @@ -74,8 +109,8 @@ spec: } } - // Patch the workload pod: token volume, init-wait container, creds env on the - // app container. + // Patch the workload pod: token volume (for app creds), a PodIdentity-readiness + // init gate, and creds env on the app container. patch: spec: template: spec: { // +patchKey=name volumes: [{ @@ -89,12 +124,18 @@ spec: }] }] // +patchKey=name + // Control-plane readiness gate: distroless kubectl (entrypoint = kubectl) + // blocks until the PodIdentity resource reports Ready. Uses the pod's + // ServiceAccount (in-cluster config) + the Role/RoleBinding emitted above. initContainers: [{ - name: "wait-for-aws-identity" + name: "wait-for-pod-identity" image: parameter.waitImage - command: ["sh", "-c", "until aws sts get-caller-identity >/dev/null 2>&1; do echo 'waiting for AWS pod identity...'; sleep 2; done; echo 'AWS identity ready'"] - env: _credsEnv - volumeMounts: [_tokenMount] + args: [ + "wait", "--for=condition=Ready", + "podidentities.platform.gitops.io/\(context.name)", + "-n", context.namespace, + "--timeout=300s", + ] }] // +patchKey=name containers: [{ diff --git a/platform/oam/definitions/traits/aws-service-identity.cue b/platform/oam/definitions/traits/aws-service-identity.cue index 44727d99..74677d7c 100644 --- a/platform/oam/definitions/traits/aws-service-identity.cue +++ b/platform/oam/definitions/traits/aws-service-identity.cue @@ -10,9 +10,19 @@ // Ordering: EKS injects Pod Identity creds via a mutating webhook at pod // admission, which only fires if the association already exists. To avoid that // race we SELF-INJECT the creds URI + the pods.eks.amazonaws.com projected -// token and add an init container that blocks until `aws sts get-caller-identity` -// succeeds. Verified: the EKS webhook skips injection when the creds URI env is -// already present (no duplicate volume), and STS resolves without a region env. +// token, and add an init container that blocks (via `kubectl wait +// --for=condition=Ready`) until the PodIdentity resource is Ready — i.e. the +// Composition has created the IAM Role + PodIdentityAssociation. +// +// NOTE: this gate is CONTROL-PLANE readiness only. The brief AWS IAM data-plane +// propagation window (e.g. freshly-attached accessFor policies not yet enforced) +// is intentionally NOT handled here — handle it with app-level retry. +// +// The init container reads the PodIdentity via the pod's ServiceAccount, so the +// trait also emits a namespaced Role/RoleBinding granting get/list/watch on it. +// The image is distroless (no shell); `kubectl wait` is a single invocation, and +// if the PodIdentity isn't created yet the init container fails and the kubelet +// retries it (init-container restart) until it exists and is Ready. // // Cloud-agnostic sibling pattern: gcp-service-identity / azure-service-identity. "aws-service-identity": { @@ -35,8 +45,9 @@ template: { accessFor?: [...string] // +usage=Container to inject AWS credentials into (defaults to the component name) containerName: *context.name | string - // +usage=Image for the init container that waits for AWS identity readiness - waitImage: *"public.ecr.aws/aws-cli/aws-cli:latest" | string + // +usage=Distroless kubectl image for the PodIdentity-readiness init gate (entrypoint = kubectl). + // Chainguard kubectl:latest, pinned by multi-arch index digest (amd64+arm64) for immutability. + waitImage: *"public.ecr.aws/chainguard/kubectl:latest@sha256:5cd49041fed950723afaefcd141a163e5a5306f243841510d3e1e3667b0cdfb9" | string } // Self-injected creds URI + token mount (the EKS webhook skips injection when @@ -68,6 +79,40 @@ template: { } } + // RBAC so the pod's ServiceAccount can read its own PodIdentity resource + // (used by the wait-for-pod-identity init container). Namespaced, read-only. + "\(context.name)-podidentity-reader-role": { + apiVersion: "rbac.authorization.k8s.io/v1" + kind: "Role" + metadata: { + name: "\(context.name)-podidentity-reader" + namespace: context.namespace + } + rules: [{ + apiGroups: ["platform.gitops.io"] + resources: ["podidentities"] + verbs: ["get", "list", "watch"] + }] + } + "\(context.name)-podidentity-reader-binding": { + apiVersion: "rbac.authorization.k8s.io/v1" + kind: "RoleBinding" + metadata: { + name: "\(context.name)-podidentity-reader" + namespace: context.namespace + } + roleRef: { + apiGroup: "rbac.authorization.k8s.io" + kind: "Role" + name: "\(context.name)-podidentity-reader" + } + subjects: [{ + kind: "ServiceAccount" + name: context.name + namespace: context.namespace + }] + } + // Attach sibling components' IAM policies to the role the Composition // creates (deterministic name "-role"). if parameter.accessFor != _|_ { @@ -88,8 +133,8 @@ template: { } } - // Patch the workload pod: token volume, init-wait container, creds env on the - // app container. + // Patch the workload pod: token volume (for app creds), a PodIdentity-readiness + // init gate, and creds env on the app container. patch: spec: template: spec: { // +patchKey=name volumes: [{ @@ -103,12 +148,18 @@ template: { }] }] // +patchKey=name + // Control-plane readiness gate: distroless kubectl (entrypoint = kubectl) + // blocks until the PodIdentity resource reports Ready. Uses the pod's + // ServiceAccount (in-cluster config) + the Role/RoleBinding emitted above. initContainers: [{ - name: "wait-for-aws-identity" - image: parameter.waitImage - command: ["sh", "-c", "until aws sts get-caller-identity >/dev/null 2>&1; do echo 'waiting for AWS pod identity...'; sleep 2; done; echo 'AWS identity ready'"] - env: _credsEnv - volumeMounts: [_tokenMount] + name: "wait-for-pod-identity" + image: parameter.waitImage + args: [ + "wait", "--for=condition=Ready", + "podidentities.platform.gitops.io/\(context.name)", + "-n", context.namespace, + "--timeout=300s", + ] }] // +patchKey=name containers: [{ diff --git a/platform/oam/examples/example-agent-agentcore-memory.yaml b/platform/oam/examples/example-agent-agentcore-memory.yaml index 4c20eb6c..1e3ba151 100644 --- a/platform/oam/examples/example-agent-agentcore-memory.yaml +++ b/platform/oam/examples/example-agent-agentcore-memory.yaml @@ -37,6 +37,15 @@ spec: properties: description: "Assistant agent with persistent memory" systemMessage: "You are a helpful assistant." + # CPU request guarantees a fair cgroup CPU share + drives scheduler spread, + # so the CPU-heavy startup isn't starved on a contended node (which caused + # liveness-kill crashloops). No CPU limit on purpose (avoids startup throttling). + resources: + requests: + cpu: "500m" + memory: "512Mi" + limits: + memory: "1Gi" memory: provider: agentcore config: From 73417c61e4ee1b121c1c71e8090eeab06a14b278 Mon Sep 17 00:00:00 2001 From: shapirov103 Date: Tue, 21 Jul 2026 18:03:24 -0400 Subject: [PATCH 3/4] fix(strands-agent-base): CVE remediation + AccessDenied retry, A2A factory fix Dependency/base-image hardening: - Pin fastapi==0.139.2, starlette==1.3.1 (fixes CVE-2025-62727, CVE-2026-48818, CVE-2026-54283), pydantic==2.13.4, uvicorn==0.51.0, bedrock-agentcore==1.18.1 (previously unpinned), aws-opentelemetry-distro==0.18.0, strands-agents==1.48.0. - Add tenacity==9.1.4 for retry-with-backoff (replaces hand-rolled loop). - Pin Dockerfile builder (uv:0.9.30-python3.13-bookworm-slim) and final stage (python:3.13-slim-bookworm by digest) for supply-chain immutability. - Remaining OS-layer CVEs (zlib1g, perl, libsqlite3-0, util-linux) are Debian bookworm will_not_fix/fix_deferred/no-dsa - not resolvable via apt upgrade. Resilience: - agent.py: tenacity retry() on _construct_agent, retrying only on botocore AccessDeniedException (first-boot IAM/Pod-Identity propagation race), exponential backoff (multiplier=1, max=16s), capped at 6 attempts. - agent.py: skip AgentCore session-manager attachment for the A2A agent-card placeholder context id (__agent_card__) - that probe agent is never used for real requests, and the placeholder fails AgentCore's sessionId validation regex (must start with alphanumeric). - main.py: switch A2AServer from the deprecated single shared agent param (not multi-tenant-safe, and rejected by strands-agents 1.48's StrandsA2AExecutor when a session_manager is attached) to the agent_factory param, giving each A2A context its own context-scoped agent + session_manager. Verified: rebuilt multi-arch (amd64+arm64), re-scanned with trivy (starlette/ fastapi/tenacity/etc. all 0 vulnerabilities), pushed to public.ecr.aws/z0a4o2j5/strands-agent (:latest, :v1.1.0-security), and confirmed on the hub cluster - all 3 my-agent pods Running 1/1, 0 restarts, Rollout Healthy. --- applications/strands-agent-base/Dockerfile | 4 +- applications/strands-agent-base/app/agent.py | 54 ++++++++++++++++--- applications/strands-agent-base/app/main.py | 13 ++--- .../strands-agent-base/pyproject.toml | 14 ++--- 4 files changed, 63 insertions(+), 22 deletions(-) diff --git a/applications/strands-agent-base/Dockerfile b/applications/strands-agent-base/Dockerfile index 7f65b2bc..4c2a9d23 100644 --- a/applications/strands-agent-base/Dockerfile +++ b/applications/strands-agent-base/Dockerfile @@ -1,5 +1,5 @@ # Use uv's Python base image for fast dependency installation -FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim AS builder +FROM ghcr.io/astral-sh/uv:0.9.30-python3.13-bookworm-slim AS builder WORKDIR /app @@ -10,7 +10,7 @@ COPY pyproject.toml ./ RUN uv pip install --system --no-cache --prerelease=allow . # Final stage -FROM python:3.13-slim-bookworm +FROM python:3.13-slim-bookworm@sha256:9d7f287598e1a5a978c015ee176d8216435aaf335ed69ac3c38dd1bbb10e8d64 # Patch CVEs in base image system packages RUN apt-get update && \ diff --git a/applications/strands-agent-base/app/agent.py b/applications/strands-agent-base/app/agent.py index c9502b92..4283a270 100644 --- a/applications/strands-agent-base/app/agent.py +++ b/applications/strands-agent-base/app/agent.py @@ -11,10 +11,19 @@ format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", ) +from botocore.exceptions import ClientError from mcp.client.streamable_http import streamablehttp_client from strands import Agent from strands.models.openai import OpenAIModel from strands.tools.mcp.mcp_client import MCPClient +from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential, before_sleep_log + +try: + from strands.multiagent.a2a.server import _AGENT_CARD_CONTEXT_ID +except ImportError: + # Fallback if the SDK renames/removes this internal constant; matches + # the value as of strands-agents 1.48.0. + _AGENT_CARD_CONTEXT_ID = "__agent_card__" from .config import config @@ -27,6 +36,11 @@ _mcp_exit_stack: Optional[ExitStack] = None +def _is_access_denied(exc: BaseException) -> bool: + """True if *exc* is a botocore AccessDeniedException (any service).""" + return isinstance(exc, ClientError) and exc.response.get("Error", {}).get("Code") == "AccessDeniedException" + + def _get_model() -> OpenAIModel: global _model if _model is None: @@ -100,6 +114,14 @@ def _build_session_manager(session_id: str, actor_id: str): if config.MEMORY_PROVIDER != "agentcore": return None + # The A2AServer agent_factory is invoked once at construction with a + # placeholder context id ("__agent_card__") solely to derive agent-card + # metadata; that agent is never used for request handling. Skip memory + # attachment for it — AgentCore session ids must start with an + # alphanumeric character, which the placeholder does not satisfy. + if session_id == _AGENT_CARD_CONTEXT_ID: + return None + mem_config = config.MEMORY_CONFIG memory_id = mem_config.get("memoryId") region = mem_config.get("region", config.AWS_REGION) @@ -124,18 +146,23 @@ def _build_session_manager(session_id: str, actor_id: str): return sm -def create_agent(session_id: Optional[str] = None, actor_id: str = "user") -> Agent: - """Create a Strands agent for a given session. +@retry( + retry=retry_if_exception(_is_access_denied), + wait=wait_exponential(multiplier=1, max=16), + stop=stop_after_attempt(6), + before_sleep=before_sleep_log(logger, logging.WARNING), + reraise=True, +) +def _construct_agent(session_id: str, actor_id: str) -> Agent: + """Build the session manager + Agent. - Args: - session_id: Conversation session id. A new UUID is generated when None. - actor_id: Identity of the caller (default "user"). + Retries on AccessDeniedException (first-boot IAM propagation race + between Pod Identity association and the AgentCore access policy) + with exponential backoff instead of crashing the process. """ - session_id = session_id or str(uuid.uuid4()) session_manager = _build_session_manager(session_id, actor_id) tools = _get_mcp_tools() or None - - agent = Agent( + return Agent( model=_get_model(), system_prompt=config.SYSTEM_PROMPT, tools=tools, @@ -144,6 +171,17 @@ def create_agent(session_id: Optional[str] = None, actor_id: str = "user") -> Ag description=config.AGENT_DESCRIPTION, session_manager=session_manager, ) + + +def create_agent(session_id: Optional[str] = None, actor_id: str = "user") -> Agent: + """Create a Strands agent for a given session. + + Args: + session_id: Conversation session id. A new UUID is generated when None. + actor_id: Identity of the caller (default "user"). + """ + session_id = session_id or str(uuid.uuid4()) + agent = _construct_agent(session_id, actor_id) logger.info(f"Agent created: {config.AGENT_NAME} session={session_id}") return agent diff --git a/applications/strands-agent-base/app/main.py b/applications/strands-agent-base/app/main.py index 524b5e9a..8fd046cc 100644 --- a/applications/strands-agent-base/app/main.py +++ b/applications/strands-agent-base/app/main.py @@ -55,13 +55,14 @@ async def lifespan(app): shutdown_mcp() -# A2A server needs an agent for the agent card / default executor. -# Per-session routing for A2A would require a custom executor; for now -# the default A2A executor uses this shared agent (no memory). -_default_agent = create_agent() - +# A2AServer builds one Agent per A2A context via agent_factory (context_id -> +# Agent), so each caller/session gets its own AgentCore-backed session_manager +# instead of all A2A callers sharing a single memory-less agent. create_agent's +# signature (session_id, actor_id="user") matches (context_id) -> Agent when +# called positionally. The factory is invoked once up front (with a placeholder +# context id) purely to derive agent-card metadata. a2a_server = A2AServer( - agent=_default_agent, + agent_factory=create_agent, host=config.HOST, port=config.PORT, version="1.0.0", diff --git a/applications/strands-agent-base/pyproject.toml b/applications/strands-agent-base/pyproject.toml index 915b021b..ce3f726b 100644 --- a/applications/strands-agent-base/pyproject.toml +++ b/applications/strands-agent-base/pyproject.toml @@ -4,12 +4,14 @@ version = "1.0.0" description = "Strands Agent with A2A protocol support" requires-python = ">=3.11" dependencies = [ - "fastapi~=0.115.0", - "uvicorn[standard]>=0.34.2", - "pydantic~=2.0", - "strands-agents[a2a,openai,otel]~=1.0", - "bedrock-agentcore", - "aws-opentelemetry-distro>=0.18.0", + "fastapi==0.139.2", + "starlette==1.3.1", + "uvicorn[standard]==0.51.0", + "pydantic==2.13.4", + "strands-agents[a2a,openai,otel]==1.48.0", + "bedrock-agentcore==1.18.1", + "aws-opentelemetry-distro==0.18.0", + "tenacity==9.1.4", ] [project.optional-dependencies] From 7f4f925ea92c83d682fae6aa3c5833fbdf232caf Mon Sep 17 00:00:00 2001 From: shapirov103 Date: Fri, 31 Jul 2026 16:05:28 -0400 Subject: [PATCH 4/4] fix(strands-agent-base): recycle MCP connections before the gateway token expires The MCP client opened one persistent connection per server at first use and reused it indefinitely (_mcp_exit_stack), authenticated with the gateway-identity workload token read at connect time. That token has a fixed 1h TTL (expirationSeconds=3600 on the projected ServiceAccount token). The kubelet rotates the token file on disk, but the already-open MCP connection never re-reads it, so any tool call issued after ~1h of pod uptime failed with a 401 from agentgateway ("Error(ExpiredSignature)"), surfacing to the user as "unable to access the time tool due to a connection issue." Fix: track each MCPClient instance (not just an ExitStack) and, once a connection exceeds _MCP_CONNECTION_MAX_AGE_SECONDS (45 min, under the 1h TTL), call stop() then start() on the SAME instance rather than creating a new one. This re-invokes the transport callable (and therefore _gateway_headers(), which reads the token file fresh) while preserving object identity - required because MCPClient.call_tool_async is bound to self and tool objects returned by list_tools_sync() delegate through the client instance, not a frozen session. Any already-cached Agent (the session cache in get_or_create_agent is unbounded and never evicted) keeps working after a recycle instead of holding a reference to a client that would otherwise need replacing. Verified via a mocked unit test: after forcing a near-zero max age, the second call recycles via stop()+start() on the identical object (checked with `is`), not a new instance, with the tools list unchanged. Rebuilt and pushed a multi-arch image (public.ecr.aws/z0a4o2j5/strands-agent :latest, :v1.2.0-mcp-recycle) and redeployed both default/oap-assistant-a and agents/my-agent; confirmed a live mcp-time tool call succeeds end-to-end with no regression. --- applications/strands-agent-base/app/agent.py | 61 ++++++++++++++++---- 1 file changed, 50 insertions(+), 11 deletions(-) diff --git a/applications/strands-agent-base/app/agent.py b/applications/strands-agent-base/app/agent.py index 4283a270..4d1b2235 100644 --- a/applications/strands-agent-base/app/agent.py +++ b/applications/strands-agent-base/app/agent.py @@ -2,8 +2,8 @@ import logging import os +import time import uuid -from contextlib import ExitStack from typing import Optional logging.basicConfig( @@ -33,7 +33,24 @@ _model: Optional[OpenAIModel] = None _mcp_tools: list = [] -_mcp_exit_stack: Optional[ExitStack] = None +_mcp_clients: list = [] +_mcp_connected_at: Optional[float] = None + +# The gateway-identity token (audience "agentgateway") mounted at +# WORKLOAD_TOKEN_PATH has a fixed TTL (expirationSeconds on the projected +# ServiceAccount token, currently 1h). _get_mcp_tools() opens a persistent +# MCP connection per server and reuses it, so a long-lived agent session +# eventually calls a tool with the credentials the connection authenticated +# with at connect time — which expire even though the token *file* on disk +# gets rotated by the kubelet, because the open connection doesn't re-read +# it. Recycle each MCPClient in place (stop() + start() on the same +# instance, which re-invokes the transport callable and therefore +# _gateway_headers()) after this many seconds, well under the token's 1h +# lifetime. Reconnecting the same instances (rather than creating new ones) +# keeps any already-built Agent's cached tool objects valid, since those +# tools are bound to the MCPClient object identity, not a point-in-time +# session. +_MCP_CONNECTION_MAX_AGE_SECONDS = 45 * 60 def _is_access_denied(exc: BaseException) -> bool: @@ -81,29 +98,47 @@ def _gateway_headers() -> dict: def _get_mcp_tools() -> list: - global _mcp_tools, _mcp_exit_stack - if _mcp_exit_stack is not None: + global _mcp_tools, _mcp_clients, _mcp_connected_at + + if _mcp_clients: + age = time.monotonic() - _mcp_connected_at + if age < _MCP_CONNECTION_MAX_AGE_SECONDS: + return _mcp_tools + logger.info( + "Recycling %d MCP connection(s) after %.0fs (max age %ds) so the " + "gateway auth token is re-read fresh", + len(_mcp_clients), age, _MCP_CONNECTION_MAX_AGE_SECONDS, + ) + for client in _mcp_clients: + try: + client.stop(None, None, None) + client.start() + except Exception as exc: + logger.warning(f" Failed to recycle MCP connection: {exc}") + _mcp_connected_at = time.monotonic() return _mcp_tools urls = config.MCP_SERVER_URLS if not urls: return [] - stack = ExitStack() + clients: list = [] tools: list = [] for url in urls: logger.info(f"Connecting to MCP server: {url}") try: client = MCPClient(lambda u=url: streamablehttp_client(u, headers=_gateway_headers())) - stack.enter_context(client) + client.start() server_tools = client.list_tools_sync() logger.info(f" Loaded {len(server_tools)} tools from {url}") + clients.append(client) tools.extend(server_tools) except Exception as exc: logger.warning(f" Failed to connect to MCP server {url}: {exc}") _mcp_tools = tools - _mcp_exit_stack = stack + _mcp_clients = clients + _mcp_connected_at = time.monotonic() return _mcp_tools @@ -208,8 +243,12 @@ def get_or_create_agent(session_id: Optional[str] = None, actor_id: str = "user" # ── cleanup ────────────────────────────────────────────────────────────── def shutdown_mcp() -> None: - global _mcp_exit_stack - if _mcp_exit_stack is not None: + global _mcp_clients + if _mcp_clients: logger.info("Closing MCP client connections") - _mcp_exit_stack.close() - _mcp_exit_stack = None + for client in _mcp_clients: + try: + client.stop(None, None, None) + except Exception as exc: + logger.warning(f" Failed to close MCP connection: {exc}") + _mcp_clients = []