diff --git a/apps/api/src/config/env-boolean.test.ts b/apps/api/src/config/env-boolean.test.ts new file mode 100644 index 0000000..26bd2e6 --- /dev/null +++ b/apps/api/src/config/env-boolean.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from 'vitest'; +import { z } from 'zod'; +import { envBoolean } from './env.js'; + +/** + * #325 — `LANGFUSE_ALLOW_REMOTE_PAYLOADS` was `z.coerce.boolean()`, i.e. + * `Boolean(value)`. Because an env var is ALWAYS a string, every non-empty value + * is truthy: `=false`, `=0` and `=no` all evaluated to **true**. + * + * That inverts the operator's intent on a §2.2 security gate — writing + * `LANGFUSE_ALLOW_REMOTE_PAYLOADS=false` to keep memory text on-box would have + * started shipping prompts and completions to a remote Langfuse. + */ + +describe('envBoolean', () => { + const schema = envBoolean.default(false); + + it('reads the falsey spellings as FALSE — the whole point of the fix', () => { + // Under `z.coerce.boolean()` every one of these was `true`. + for (const raw of ['false', '0', 'no', 'off', '']) { + expect(schema.parse(raw), `"${raw}" must be false`).toBe(false); + } + }); + + it('reads the truthy spellings as TRUE', () => { + for (const raw of ['true', '1', 'yes', 'on']) { + expect(schema.parse(raw), `"${raw}" must be true`).toBe(true); + } + }); + + it('is case- and whitespace-insensitive', () => { + expect(schema.parse(' FALSE ')).toBe(false); + expect(schema.parse('True')).toBe(true); + }); + + it('defaults to false when the variable is absent', () => { + expect(schema.parse(undefined)).toBe(false); + }); + + it('rejects an unrecognised value rather than guessing', () => { + // Failing closed at boot beats a security gate whose state nobody can predict. + for (const raw of ['maybe', 'FALSE!', 'y', '2']) { + expect(() => schema.parse(raw), `"${raw}" must throw`).toThrow(); + } + }); + + it('names the accepted spellings in the error, so the fix is obvious', () => { + const result = z.object({ FLAG: envBoolean }).safeParse({ FLAG: 'maybe' }); + expect(result.success).toBe(false); + if (!result.success) { + expect(result.error.issues[0]?.message).toContain('true'); + expect(result.error.issues[0]?.message).toContain('maybe'); + } + }); +}); diff --git a/apps/api/src/config/env.ts b/apps/api/src/config/env.ts index a1caccf..946e029 100644 --- a/apps/api/src/config/env.ts +++ b/apps/api/src/config/env.ts @@ -1,6 +1,32 @@ import 'dotenv/config'; import { z } from 'zod'; +/** + * A boolean read from an environment variable, where the value is always a + * string. Only the listed spellings are accepted; anything else is a hard parse + * error rather than a silent guess. + * + * Deliberately NOT `z.coerce.boolean()`, which is `Boolean(value)` and therefore + * true for EVERY non-empty string — `=false`, `=0` and `=no` would all mean true + * (#325). On a security gate that inverts the operator's intent silently. + */ +const TRUE_VALUES = new Set(['1', 'true', 'yes', 'on']); +const FALSE_VALUES = new Set(['0', 'false', 'no', 'off', '']); + +export const envBoolean = z + .string() + .transform((raw, ctx) => { + const value = raw.trim().toLowerCase(); + if (TRUE_VALUES.has(value)) return true; + if (FALSE_VALUES.has(value)) return false; + ctx.addIssue({ + code: 'custom', + message: `expected one of ${[...TRUE_VALUES, ...FALSE_VALUES].filter(Boolean).join(', ')} — got "${raw}"`, + }); + return z.NEVER; + }) + .pipe(z.boolean()); + /** * Environment is validated once at boot. Importing `env` anywhere guarantees * a fully-typed, present-and-correct configuration or a hard fail on startup. @@ -131,7 +157,12 @@ const envSchema = z.object({ // Langfuse — an explicit, auditable opt-in; otherwise a non-loopback host // degrades to metadata-only (model, tokens, cost, latency) rather than // silently shipping memory text off-box (failure mode M9). - LANGFUSE_ALLOW_REMOTE_PAYLOADS: z.coerce.boolean().default(false), + // + // NOT `z.coerce.boolean()` (#325): that is `Boolean(string)`, so every + // non-empty value is true — `=false`, `=0` and `=no` would all ENABLE remote + // payloads. An operator writing `=false` to turn the escape hatch off would + // have turned it on, silently, on a security gate. + LANGFUSE_ALLOW_REMOTE_PAYLOADS: envBoolean.default(false), }); /** diff --git a/apps/api/src/platform/model-router/index.ts b/apps/api/src/platform/model-router/index.ts index 15d92ec..1e0136a 100644 --- a/apps/api/src/platform/model-router/index.ts +++ b/apps/api/src/platform/model-router/index.ts @@ -229,6 +229,18 @@ export interface CompleteArgs { * per-class estimate is used. Tests drive spend over the cap by injecting this. */ readonly cost?: number; + /** + * Marks this call as carrying PRIVATE memory content (#325, invariant §2.2). + * When set, the Langfuse trace's input/output are redacted unconditionally — + * even on loopback, even with the remote opt-in. + * + * Recall does not set it because it pre-filters private memories out of + * `contexts` upstream (`publicHits` in memory.service.ts), so its prompts carry + * no private text. Any FUTURE routed call that does handle private content must + * set this; without it `safePayload`'s unconditional-redaction branch is + * unreachable and the guarantee is only a comment. + */ + readonly isPrivate?: boolean; } /** @@ -279,9 +291,11 @@ function traceGeneration(input: { costUsd: number; latencyMs: number; startedAt: Date; + isPrivate?: boolean; }): void { const lf = getLangfuse(); if (!lf) return; + const redact = { isPrivate: input.isPrivate }; safely(() => { const trace = lf.trace({ name: `llm.${input.modelClass}`, @@ -290,8 +304,8 @@ function traceGeneration(input: { // the trace root is what the UI shows first, and leaving them unset // renders "this trace didn't receive an input or output" even though the // generation underneath has both. - input: safePayload(input.promptText), - output: safePayload(input.answerText), + input: safePayload(input.promptText, redact), + output: safePayload(input.answerText, redact), metadata: { route: input.route, vendor: input.vendor, costUsd: input.costUsd }, }); trace.generation({ @@ -299,8 +313,8 @@ function traceGeneration(input: { model: input.model, startTime: input.startedAt, endTime: new Date(input.startedAt.getTime() + input.latencyMs), - input: safePayload(input.promptText), - output: safePayload(input.answerText), + input: safePayload(input.promptText, redact), + output: safePayload(input.answerText, redact), metadata: { modelClass: input.modelClass, vendor: input.vendor, @@ -335,6 +349,7 @@ export async function complete( traceGeneration({ modelClass: args.class, model, vendor, route, promptText, answerText: result.answer, costUsd: cost, latencyMs, startedAt, + isPrivate: args.isPrivate, }); return result; } @@ -366,6 +381,7 @@ export async function complete( traceGeneration({ modelClass: args.class, model, vendor, route, userId: args.userId, promptText, answerText: result.answer, costUsd: spend, latencyMs, startedAt, + isPrivate: args.isPrivate, }); // Record spend only AFTER a successful provider call (reconciled on threshold-cross). diff --git a/apps/api/src/platform/model-router/langfuse-trace.test.ts b/apps/api/src/platform/model-router/langfuse-trace.test.ts new file mode 100644 index 0000000..033e993 --- /dev/null +++ b/apps/api/src/platform/model-router/langfuse-trace.test.ts @@ -0,0 +1,146 @@ +import type { Langfuse } from 'langfuse'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { setLangfuseClient } from '../observability/langfuse.js'; +import { complete } from './index.js'; + +/** + * #325 — the spy-provider half of the §2.2 proof for Langfuse tracing (#320 + * stage 2). The unit tests in `observability/langfuse.test.ts` prove the gating + * FUNCTIONS behave; these prove the routed `complete()` path actually applies + * them, and that a broken Langfuse can never break a recall. + * + * Hermetic: `keys: {}` forces the deterministic FAKE provider, so nothing here + * touches the network. The Langfuse client is a spy injected through the module's + * test seam — tracing is otherwise (correctly) off in tests. + */ + +/** Everything the spy was handed, so a test can search it for leaked content. */ +interface Captured { + traces: unknown[]; + generations: unknown[]; +} + +function spyClient(opts: { throwOnTrace?: boolean } = {}): { + client: Langfuse; + captured: Captured; +} { + const captured: Captured = { traces: [], generations: [] }; + const client = { + trace: (body: unknown) => { + if (opts.throwOnTrace) throw new Error('langfuse is down'); + captured.traces.push(body); + return { + generation: (genBody: unknown) => { + captured.generations.push(genBody); + return {}; + }, + }; + }, + } as unknown as Langfuse; + return { client, captured }; +} + +/** Every string the spy received, flattened — what "reached Langfuse" means. */ +const everythingSent = (captured: Captured): string => + JSON.stringify([...captured.traces, ...captured.generations]); + +const PRIVATE_TEXT = 'my HIV test result came back positive'; +const PUBLIC_TEXT = 'the standup moved to 9:30'; + +afterEach(() => { + setLangfuseClient(undefined); + vi.restoreAllMocks(); +}); + +describe('AC3: private memory text never reaches the Langfuse client', () => { + it('redacts input AND output when the routed call is marked private', async () => { + const { client, captured } = spyClient(); + setLangfuseClient(client); + + await complete( + { + class: 'fast', + query: 'what did the clinic say?', + contexts: [{ id: 'm1', text: PRIVATE_TEXT }], + isPrivate: true, + }, + {}, + ); + + // The spy must have been called — otherwise this test proves nothing. + expect(captured.traces).toHaveLength(1); + expect(captured.generations).toHaveLength(1); + + const sent = everythingSent(captured); + expect(sent).not.toContain('HIV'); + expect(sent).not.toContain(PRIVATE_TEXT); + expect(sent).toContain('private-memory'); + }); + + it('redacts the private prompt even though the host is loopback', async () => { + // Loopback normally ALLOWS payloads; `isPrivate` must override that. + const { client, captured } = spyClient(); + setLangfuseClient(client); + + await complete( + { + class: 'fast', + query: 'clinic', + contexts: [{ id: 'm1', text: PRIVATE_TEXT }], + isPrivate: true, + }, + {}, + ); + + expect(everythingSent(captured)).not.toContain('HIV'); + }); + + it('still traces non-private content, so the redaction is not vacuous', async () => { + // If the spy saw nothing either way, the assertions above would be worthless. + const { client, captured } = spyClient(); + setLangfuseClient(client); + + await complete( + { class: 'fast', query: 'when is standup?', contexts: [{ id: 'm1', text: PUBLIC_TEXT }] }, + {}, + ); + + expect(everythingSent(captured)).toContain('standup'); + }); +}); + +describe('AC4: tracing is fail-open', () => { + it('a throwing Langfuse client does not fail or reject the LLM call', async () => { + const { client } = spyClient({ throwOnTrace: true }); + setLangfuseClient(client); + + const result = await complete( + { class: 'fast', query: 'when is standup?', contexts: [{ id: 'm1', text: PUBLIC_TEXT }] }, + {}, + ); + + // The answer still comes back — observability cannot break a recall. + expect(result.answer).toBeTypeOf('string'); + expect(result.answer.length).toBeGreaterThan(0); + }); + + it('swallows the trace error rather than propagating it', async () => { + const { client } = spyClient({ throwOnTrace: true }); + setLangfuseClient(client); + + await expect( + complete({ class: 'fast', query: 'q', contexts: [{ id: 'm1', text: PUBLIC_TEXT }] }, {}), + ).resolves.toBeDefined(); + }); + + it('is a no-op when tracing is disabled (no client)', async () => { + setLangfuseClient(undefined); + + const result = await complete( + { class: 'fast', query: 'when is standup?', contexts: [{ id: 'm1', text: PUBLIC_TEXT }] }, + {}, + ); + + expect(result.answer).toBeTypeOf('string'); + }); +}); diff --git a/apps/api/src/platform/observability/langfuse.test.ts b/apps/api/src/platform/observability/langfuse.test.ts new file mode 100644 index 0000000..8d66449 --- /dev/null +++ b/apps/api/src/platform/observability/langfuse.test.ts @@ -0,0 +1,138 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + type TracingConfig, + getLangfuse, + isTracingEnabled, + payloadsAllowed, + safePayload, + safely, +} from './langfuse.js'; + +/** + * #325 — the Langfuse privacy gating (#320 stage 2) had NO test coverage. A trace's + * input/output are the prompt and the completion, i.e. MEMORY TEXT, so a + * regression here duplicates memory content into another store — exactly what + * invariant §2.2 forbids and failure mode M9 describes. + * + * Config is injected rather than mocked, matching `resolve(class, keys)` in the + * model router: the suite never reads global env and never touches the network. + */ + +const LOOPBACK: TracingConfig = { + baseUrl: 'http://localhost:3002', + allowRemotePayloads: false, +}; +const REMOTE: TracingConfig = { + baseUrl: 'https://cloud.langfuse.com', + allowRemotePayloads: false, +}; +const REMOTE_OPTED_IN: TracingConfig = { + baseUrl: 'https://cloud.langfuse.com', + allowRemotePayloads: true, +}; + +const MEMORY_TEXT = 'the therapist appointment is on Tuesday at 4pm'; + +describe('payloadsAllowed', () => { + it('allows payloads on a loopback host (the self-hosted default)', () => { + expect(payloadsAllowed(LOOPBACK)).toBe(true); + expect(payloadsAllowed({ ...LOOPBACK, baseUrl: 'http://127.0.0.1:3002' })).toBe(true); + expect(payloadsAllowed({ ...LOOPBACK, baseUrl: 'http://[::1]:3002' })).toBe(true); + }); + + it('refuses payloads on a non-loopback host with no explicit opt-in', () => { + expect(payloadsAllowed(REMOTE)).toBe(false); + }); + + it('allows payloads on a remote host ONLY with the explicit opt-in', () => { + expect(payloadsAllowed(REMOTE_OPTED_IN)).toBe(true); + }); + + it('treats an unparseable base URL as remote (fails closed)', () => { + expect(payloadsAllowed({ ...LOOPBACK, baseUrl: 'not a url' })).toBe(false); + }); + + it('does not treat a host merely CONTAINING localhost as loopback', () => { + // `localhost.evil.com` resolves wherever the attacker wants it to. + expect(payloadsAllowed({ ...LOOPBACK, baseUrl: 'http://localhost.evil.com/' })).toBe(false); + }); +}); + +describe('safePayload', () => { + it('redacts isPrivate content UNCONDITIONALLY — even on loopback', () => { + // The strongest rule in the module: a private memory's text must never be + // duplicated into another store, and a trace IS another store. + expect(safePayload(MEMORY_TEXT, { isPrivate: true }, LOOPBACK)).toEqual({ + redacted: 'private-memory', + }); + }); + + it('redacts isPrivate content even WITH the remote opt-in set', () => { + expect(safePayload(MEMORY_TEXT, { isPrivate: true }, REMOTE_OPTED_IN)).toEqual({ + redacted: 'private-memory', + }); + }); + + it('redacts on a remote host when the opt-in is absent', () => { + expect(safePayload(MEMORY_TEXT, {}, REMOTE)).toEqual({ redacted: 'remote-host' }); + }); + + it('passes the payload through on loopback for non-private content', () => { + expect(safePayload(MEMORY_TEXT, {}, LOOPBACK)).toBe(MEMORY_TEXT); + }); + + it('passes the payload through on a remote host with the explicit opt-in', () => { + expect(safePayload(MEMORY_TEXT, {}, REMOTE_OPTED_IN)).toBe(MEMORY_TEXT); + }); + + it('never returns the original text in any redacting branch', () => { + // Belt-and-braces: whatever the redaction shape is, it must not carry content. + for (const cfg of [LOOPBACK, REMOTE, REMOTE_OPTED_IN]) { + const redacted = safePayload(MEMORY_TEXT, { isPrivate: true }, cfg); + expect(JSON.stringify(redacted)).not.toContain('therapist'); + } + expect(JSON.stringify(safePayload(MEMORY_TEXT, {}, REMOTE))).not.toContain('therapist'); + }); +}); + +describe('isTracingEnabled', () => { + it('is off when neither key is set — dev/CI/tests never emit', () => { + expect(isTracingEnabled(LOOPBACK)).toBe(false); + }); + + it('is off when only one of the two keys is set', () => { + expect(isTracingEnabled({ ...LOOPBACK, publicKey: 'pk-only' })).toBe(false); + expect(isTracingEnabled({ ...LOOPBACK, secretKey: 'sk-only' })).toBe(false); + }); + + it('is on only when both keys are present', () => { + expect(isTracingEnabled({ ...LOOPBACK, publicKey: 'pk', secretKey: 'sk' })).toBe(true); + }); +}); + +describe('AC5: the suite itself never emits', () => { + it('has tracing disabled under the test env, whatever is in the local .env', () => { + // Reads the REAL process env deliberately (no injected config): this asserts + // the harness, not the function. `vitest.config.ts` blanks both Langfuse keys + // so a developer running a local Langfuse does not ship traces of test data + // into it — the same fake-by-default posture as every other provider (§2.5). + expect(isTracingEnabled()).toBe(false); + expect(getLangfuse()).toBeUndefined(); + }); +}); + +describe('safely', () => { + it('swallows a throwing emit so observability can never fail a request', () => { + expect(() => + safely(() => { + throw new Error('langfuse exploded'); + }), + ).not.toThrow(); + }); + + it('runs the callback when it does not throw', () => { + const fn = vi.fn(); + safely(fn); + expect(fn).toHaveBeenCalledOnce(); + }); +}); diff --git a/apps/api/src/platform/observability/langfuse.ts b/apps/api/src/platform/observability/langfuse.ts index 9cf3cb8..45b89fe 100644 --- a/apps/api/src/platform/observability/langfuse.ts +++ b/apps/api/src/platform/observability/langfuse.ts @@ -31,17 +31,57 @@ import { isLoopbackUrl } from '../../modules/memory/embedding/ollama.provider.js let client: Langfuse | undefined; let initialised = false; +/** + * The knobs the gating rules read. Injected rather than read from global `env` so + * the gate can be tested exhaustively and hermetically — the same contract as + * `resolve(class, keys)` in the model router (#325). Production callers pass + * nothing and get the validated process env. + */ +export interface TracingConfig { + readonly baseUrl: string; + readonly allowRemotePayloads: boolean; + readonly publicKey?: string; + readonly secretKey?: string; +} + +/** The process-wide config, read fresh per call (env is validated at boot). */ +export function tracingConfigFromEnv(): TracingConfig { + return { + baseUrl: env.LANGFUSE_BASE_URL, + allowRemotePayloads: env.LANGFUSE_ALLOW_REMOTE_PAYLOADS, + publicKey: env.LANGFUSE_PUBLIC_KEY, + secretKey: env.LANGFUSE_SECRET_KEY, + }; +} + /** True when both keys are present — the only way tracing turns on. */ -export function isTracingEnabled(): boolean { - return Boolean(env.LANGFUSE_PUBLIC_KEY && env.LANGFUSE_SECRET_KEY); +export function isTracingEnabled(cfg: TracingConfig = tracingConfigFromEnv()): boolean { + return Boolean(cfg.publicKey && cfg.secretKey); } /** * Whether trace INPUT/OUTPUT payloads may carry content. False → metadata only. * Loopback is the safe default; a remote host requires the explicit opt-in. */ -export function payloadsAllowed(): boolean { - return isLoopbackUrl(env.LANGFUSE_BASE_URL) || env.LANGFUSE_ALLOW_REMOTE_PAYLOADS; +export function payloadsAllowed(cfg: TracingConfig = tracingConfigFromEnv()): boolean { + return isLoopbackUrl(cfg.baseUrl) || cfg.allowRemotePayloads; +} + +/** + * Override the client {@link getLangfuse} returns, and return the previous one — + * the same test-seam shape as `setBudgetTracker` in the model router (#325). + * + * Needed because tracing is correctly OFF in tests (no keys → `undefined`), so + * without a seam there is no way to assert what would have been sent. Pass + * `undefined` to restore the "tracing disabled" state. + */ +export function setLangfuseClient(next: Langfuse | undefined): Langfuse | undefined { + const previous = client; + client = next; + // Mark as initialised so `getLangfuse` returns the injected client verbatim + // instead of trying to build a real one from env. + initialised = true; + return previous; } /** The process-wide client, built once. `undefined` when tracing is off. */ @@ -81,9 +121,13 @@ export function getLangfuse(): Langfuse | undefined { * the remote opt-in. A private memory's text is the one thing that must never * be duplicated into another store, and a trace IS another store. */ -export function safePayload(value: unknown, opts: { isPrivate?: boolean } = {}): unknown { +export function safePayload( + value: unknown, + opts: { isPrivate?: boolean } = {}, + cfg: TracingConfig = tracingConfigFromEnv(), +): unknown { if (opts.isPrivate) return { redacted: 'private-memory' }; - if (!payloadsAllowed()) return { redacted: 'remote-host' }; + if (!payloadsAllowed(cfg)) return { redacted: 'remote-host' }; return value; } diff --git a/apps/api/vitest.config.ts b/apps/api/vitest.config.ts index d2d2a78..db0279a 100644 --- a/apps/api/vitest.config.ts +++ b/apps/api/vitest.config.ts @@ -36,6 +36,13 @@ export default defineConfig({ EMBEDDING_PROVIDER: 'fake', LLM_PROVIDER: 'fake', TRANSCRIPTION_PROVIDER: 'fake', + // Langfuse is keyed off the PRESENCE of both keys, so a developer with a + // local Langfuse in `.env` would otherwise have the suite build a real + // client and ship traces of test data into it (#325). Blank them: tracing + // is a no-op in tests, and the gating is asserted with injected config + + // a stubbed client instead. + LANGFUSE_PUBLIC_KEY: '', + LANGFUSE_SECRET_KEY: '', }, }, });