From 9768713e22f0b6dffba2e431684a8d38bf50ac65 Mon Sep 17 00:00:00 2001 From: Agents Agent Date: Sat, 27 Jun 2026 14:44:19 +0200 Subject: [PATCH] fix(agents-login): strip AAAA via a CoreDNS sidecar so codex sign-in uses IPv4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The /etc/hosts init container (#745) and gai.conf (#742) were both ineffective: codex's musl-static Rust client ignores /etc/hosts and /etc/gai.conf, and a hosts A-record doesn't suppress the DNS AAAA — so codex still resolved auth.openai.com's IPv6, tried it, and failed (no IPv6 egress, no IPv4 fallback). Confirmed live: even with the IPv4 hosts pin, `getent hosts auth.openai.com` returned the DNS IPv6 and `codex login --device-auth` still exited 1. Replace that approach with a per-pod CoreDNS sidecar that answers AAAA with NODATA (`template IN AAAA . { rcode NOERROR }`) and forwards everything else to kube-dns (10.43.0.10). Point the pod at it via dnsPolicy: None + dnsConfig nameserver 127.0.0.1, preserving the cluster search domains + ndots. With no AAAA ever returned, codex resolves only IPv4 (which has working egress). Resolver-agnostic — works regardless of codex bypassing libc. Needs live deploy validation (sandbox can't reach the cluster). Runtime codex in the agent-runner needs the same treatment separately. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../flux/apps/agents/agents-login/worker.yaml | 145 ++++++++++-------- 1 file changed, 77 insertions(+), 68 deletions(-) diff --git a/platform/cluster/flux/apps/agents/agents-login/worker.yaml b/platform/cluster/flux/apps/agents/agents-login/worker.yaml index 9452044b..843f5611 100644 --- a/platform/cluster/flux/apps/agents/agents-login/worker.yaml +++ b/platform/cluster/flux/apps/agents/agents-login/worker.yaml @@ -1,3 +1,28 @@ +# Pod-local resolver for the agents-login worker. Codex 0.141.x ships a musl +# static binary whose resolver ignores /etc/gai.conf and /etc/hosts for the +# failing auth.openai.com path; this CoreDNS sidecar strips AAAA at DNS level +# while forwarding every non-AAAA lookup to kube-dns. +apiVersion: v1 +kind: ConfigMap +metadata: + name: agents-login-worker-coredns + namespace: agents-system + labels: + app.kubernetes.io/name: agents-login-worker + app.kubernetes.io/part-of: agents-login +data: + Corefile: | + .:53 { + errors + health :8080 + ready :8181 + template IN AAAA . { + rcode NOERROR + } + forward . 10.43.0.10 + cache 30 + } +--- # Private half of the portal. Owns the PTY that runs `claude setup-token` and # `codex login --device`, captures credentials, and posts them to agents-api # under a Lease lock. Single instance + Recreate so the Lease is never @@ -33,6 +58,17 @@ spec: spec: serviceAccountName: agents-login-worker automountServiceAccountToken: true + dnsPolicy: None + dnsConfig: + nameservers: + - 127.0.0.1 + searches: + - agents-system.svc.cluster.local + - svc.cluster.local + - cluster.local + options: + - name: ndots + value: '5' # The image is amd64-only (it bundles the Claude/Codex CLIs + node-pty and # is not worth a QEMU arm64 build). Keep it off the arm64 Pi nodes, where # it would ImagePullBackOff on "no match for platform in manifest". @@ -45,74 +81,50 @@ spec: fsGroup: 1000 seccompProfile: type: RuntimeDefault - # Codex 0.141.x ships a musl static binary. In this cluster IPv6 egress is - # unreachable, and musl ignores /etc/gai.conf, so auth.openai.com AAAA - # answers can make `codex login --device-auth` fail before it tries IPv4. - # Resolve A records at pod startup and mount a pod-local hosts file instead - # of using static hostAliases, which would stale when Cloudflare rotates IPs. - initContainers: - - name: codex-auth-hosts - image: ghcr.io/extratoast/personal-stack/agents-login:latest - imagePullPolicy: Always - command: - - node - - -e - - | - const dns = require('node:dns').promises; - const { writeFileSync } = require('node:fs'); - - const hosts = (process.env.CODEX_IPV4_HOSTS || 'auth.openai.com') - .split(/[\s,]+/) - .filter(Boolean); - const podIp = process.env.POD_IP || '127.0.0.1'; - const hostname = process.env.HOSTNAME || 'agents-login-worker'; - const lines = [ - '# Kubernetes hosts file generated by codex-auth-hosts.', - '127.0.0.1\tlocalhost', - '::1\tlocalhost ip6-localhost ip6-loopback', - 'fe00::0\tip6-localnet', - 'fe00::0\tip6-mcastprefix', - 'fe00::1\tip6-allnodes', - 'fe00::2\tip6-allrouters', - `${podIp}\t${hostname}`, - '', - ]; - - (async () => { - for (const host of hosts) { - const ips = await dns.resolve4(host); - if (ips.length === 0) { - throw new Error(`no A records for ${host}`); - } - for (const ip of ips) { - lines.push(`${ip}\t${host}`); - } - console.log(`resolved ${host}: ${ips.join(', ')}`); - } - writeFileSync('/generated-hosts/hosts', `${lines.join('\n')}\n`, { mode: 0o644 }); - })().catch((err) => { - console.error(err); - process.exit(1); - }); - env: - - name: CODEX_IPV4_HOSTS - value: auth.openai.com - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP + containers: + - name: dns + image: registry.k8s.io/coredns/coredns:v1.12.1 + imagePullPolicy: IfNotPresent + args: + - -conf + - /etc/coredns/Corefile + ports: + - name: dns-udp + containerPort: 53 + protocol: UDP + - name: dns-tcp + containerPort: 53 + protocol: TCP securityContext: - runAsNonRoot: false - runAsUser: 0 - runAsGroup: 0 allowPrivilegeEscalation: false readOnlyRootFilesystem: true + runAsNonRoot: true capabilities: drop: ['ALL'] + add: ['NET_BIND_SERVICE'] volumeMounts: - - name: codex-auth-hosts - mountPath: /generated-hosts - containers: + - name: coredns-config + mountPath: /etc/coredns + readOnly: true + readinessProbe: + httpGet: + path: /ready + port: 8181 + initialDelaySeconds: 1 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /health + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 20 + resources: + requests: + cpu: 5m + memory: 16Mi + limits: + cpu: 100m + memory: 64Mi - name: worker image: ghcr.io/extratoast/personal-stack/agents-login:latest imagePullPolicy: Always @@ -151,10 +163,6 @@ spec: capabilities: drop: ['ALL'] volumeMounts: - - name: codex-auth-hosts - mountPath: /etc/hosts - subPath: hosts - readOnly: true - name: home mountPath: /home/agent - name: tmp @@ -177,8 +185,9 @@ spec: cpu: 500m memory: 512Mi volumes: - - name: codex-auth-hosts - emptyDir: {} + - name: coredns-config + configMap: + name: agents-login-worker-coredns - name: home emptyDir: {} - name: tmp