Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 95 additions & 2 deletions src/commands/x/provider.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,30 @@ import {
HOSTS, API_PROVIDERS, AQE_PROVIDER_TYPES, detectHosts, detectProviders,
settingsTarget, isDefault, applyHosts, applyProviders, ensureDualAgents,
undoProviders, hostInstallState, installHost, applyAqeRouter, undoAqeRouter,
bothHostsEnabled, DUAL_ROLE_TIP, JUDGE_BIAS_TIP, QE_COURT_TIP, suggestedFallbackFor,
} from '../../lib/providers.mjs';
import { loadKitConfig, saveKitConfig } from '../../lib/config.mjs';
import { ok, warn, fail, info, dim, bold } from '../../lib/output.mjs';
import { installedVersion, cmpVersions } from '../../lib/versions.mjs';
import { repoRoot } from '../../lib/paths.mjs';
import { writeJsonWithBackup } from '../../lib/settings.mjs';
import { panelFromRouting, validatePanel, readQeCourtConfig, qeCourtConfigPath, vendorOf } from '../../lib/qeCourt.mjs';

const QE_COURT_MIN_VERSION = '3.13.0';
const qeCourtShipped = () => {
const v = installedVersion('agentic-qe');
return !!v && cmpVersions(v, QE_COURT_MIN_VERSION) >= 0;
};

/** Print the dual-host guidance tips (role delegation, judge-bias, qe-court
* cross-sell) once both hosts are enabled — shared by `pick()` and
* `status()` so the strings/gating never drift between the two. */
function printDualHostTips(cfg) {
if (!bothHostsEnabled(cfg)) return;
info(DUAL_ROLE_TIP);
info(JUDGE_BIAS_TIP);
if (qeCourtShipped()) info(QE_COURT_TIP);
}

export const options = {
host: { type: 'string' }, // csv: claude,codex (pick, non-interactive)
Expand Down Expand Up @@ -117,13 +138,37 @@ async function status({ flags, cwd }) {
console.log(` ${p.id.padEnd(10)} ${key.padEnd(12)} ${conf}`);
}

printQeCourtStatus(cwd);

const codexIdle = hosts.codex.present && !cfg.providers.hosts.codex;
console.log('');
if (codexIdle) info('codex is installed but disabled — enable it with: ak x provider pick');
else ok('provider config reflects installed CLIs');
printDualHostTips(cfg);
return 0;
}

/** Read-only awareness of qe-court's per-role routing (ADR-124, aqe >= 3.13.0)
* — a third config surface alongside ruflo host env + aqe's fallback chain.
* No-op unless aqe is new enough AND the skill has already created its
* config.json (ak never creates it). */
function printQeCourtStatus(cwd) {
if (!qeCourtShipped()) return;
const root = repoRoot(cwd);
if (!root) return;
const qc = readQeCourtConfig(root);
if (!qc) return;
const panel = panelFromRouting(qc.routing);
const minVendors = qc.options?.minDistinctVendors ?? 2;
const violations = validatePanel(panel, { minVendors });
console.log(bold('\nqe-court routing') + dim(' (.claude/skills/qe-court/config.json)'));
for (const { role, provider } of panel) {
console.log(` ${role.padEnd(28)} ${provider ?? dim('(unset)')}`);
}
if (violations.length) warn(`qe-court panel invalid: ${violations.join(', ')}`);
else ok('qe-court panel valid (vendor-diverse, jury independent of writer)');
}

async function off({ cwd }) {
const cfg = loadKitConfig();
cfg.providers = { hosts: { claude: true, codex: false }, aqeProvider: null, aqeFallback: [], models: [], maxBudgetUsd: null };
Expand All @@ -139,6 +184,49 @@ const parseModels = (csv) => csv.split(',').map((s) => s.trim()).filter(Boolean)
return model ? { id, model } : { id };
});

/** Opt-in write of qe-court routing defaults (Phase C, issue #36). Only offers
* when: interactive session, aqe >= 3.13.0, the skill has already created its
* config.json (ak never creates it), and an aqeProvider was chosen. Defaults
* prosecutor.codex-review/deeperReviewer -> codex when codex is enabled, and
* jury -> aqeProvider (picking a different vendor if it would collide with
* the writer/defense). Validates the resulting panel BEFORE writing — never
* produces an invalid panel on disk. Only ever touches the `routing` key. */
async function maybeWriteQeCourtDefaults({ nonInteractive, cwd, enabled, aqeProvider }) {
if (nonInteractive || !aqeProvider) return;
if (!qeCourtShipped()) return;
const root = repoRoot(cwd);
if (!root) return;
const qc = readQeCourtConfig(root);
if (!qc) return;

const codexOn = enabled.includes('codex');
const routing = { ...(qc.routing ?? {}) };
const defenseProvider = routing.defense?.provider;
let juryProvider = aqeProvider;
let juryNote = '';
if (defenseProvider && vendorOf(juryProvider) === vendorOf(defenseProvider)) {
const alt = AQE_PROVIDER_TYPES.find((p) => vendorOf(p) !== vendorOf(defenseProvider) && p !== juryProvider);
if (alt) { juryNote = ` (switched from ${juryProvider} — same vendor as defense/writer)`; juryProvider = alt; }
}

const changes = codexOn ? [['prosecutor.codex-review', 'codex'], ['deeperReviewer', 'codex'], ['jury', juryProvider]]
: [['jury', juryProvider]];

console.log(bold('\nqe-court detected') + dim(' (.claude/skills/qe-court/config.json)'));
console.log(dim(` would set: ${changes.map(([role, p]) => `${role} → ${p}`).join(', ')}${juryNote}`));
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
const ans = (await rl.question('apply these qe-court routing defaults? [y/N]: ')).trim().toLowerCase();
rl.close();
if (ans !== 'y' && ans !== 'yes') { info('qe-court routing left unchanged'); return; }

for (const [role, provider] of changes) routing[role] = { ...(routing[role] ?? {}), provider };
const violations = validatePanel(panelFromRouting(routing), { minVendors: qc.options?.minDistinctVendors ?? 2 });
if (violations.length) { warn(`qe-court routing defaults would be invalid (${violations.join(', ')}) — not written`); return; }

writeJsonWithBackup(qeCourtConfigPath(root), { ...qc, routing });
ok(`qe-court routing updated: ${changes.map(([role, p]) => `${role}→${p}`).join(', ')}`);
}

async function pick({ flags, cwd }) {
const cfg = loadKitConfig();
const hosts = await detectHosts(cwd);
Expand Down Expand Up @@ -172,8 +260,11 @@ async function pick({ flags, cwd }) {
console.log(dim(` ${AQE_BILLING_HINT}`));
const aAns = (await rl.question(`agentic-qe primary LLM provider — ${AQE_PROVIDER_TYPES.join('/')} (blank = leave aqe default): `)).trim().toLowerCase();
aqeProvider = aAns ? aAns : null;
const fAns = (await rl.question('aqe fallback chain, ordered (e.g. "claude-code:claude-opus-4-8; openai:gpt-5.6", blank = none): ')).trim().toLowerCase();
aqeFallback = fAns ? parseFallback(fAns) : [];
const suggestion = suggestedFallbackFor(enabled);
const fAns = (await rl.question(
`aqe fallback chain, ordered (e.g. "claude-code:claude-opus-4-8; openai:gpt-5.6"${suggestion ? `, blank = use suggested [${suggestion}]` : ', blank = none'}): `,
)).trim().toLowerCase();
aqeFallback = fAns ? parseFallback(fAns) : (suggestion ? parseFallback(suggestion.toLowerCase()) : []);
const provAns = (await rl.question('ruflo API-key providers to register (e.g. openai:gpt-5.6, blank to skip): ')).trim();
if (provAns) models = parseModels(provAns);
rl.close();
Expand Down Expand Up @@ -227,5 +318,7 @@ async function pick({ flags, cwd }) {
const prov = await applyProviders(cfg, cwd);
(prov.ok ? (prov.changed ? ok : info) : warn)(`ruflo providers: ${prov.detail}`);
ok('saved to kit.json — reapplied on every `ak sync`; undo with `ak x provider off`');
await maybeWriteQeCourtDefaults({ nonInteractive, cwd, enabled, aqeProvider });
printDualHostTips(cfg);
return 0;
}
31 changes: 31 additions & 0 deletions src/lib/providers.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,37 @@ export function isDefault(cfg) {
&& (!p.aqeFallback || p.aqeFallback.length === 0);
}

/** True when both frontier hosts are enabled in persisted intent (kit.json),
* regardless of whether the env is wired yet — this is the same source
* `status()` already keys "enabled" off of. Guards the dual-mode/judge-bias
* guidance below: only relevant once both are actually opted in. */
export const bothHostsEnabled = (cfg) => !!cfg.providers?.hosts?.claude && !!cfg.providers?.hosts?.codex;

/** Guidance printed once both hosts are enabled — pointers to capability that
* already exists one layer up from ak's own wiring (see issue #36):
* - role-based dual-mode delegation lives in the separate @claude-flow/codex
* npm package's `dual` CLI, which ak installs as a prerequisite but never
* surfaces itself.
* - judge-vendor-bias: a same-vendor LLM judge scores ~8-10pp inflated versus
* a cross-vendor judge (still ordinally correct, not calibrated) — measured
* in openrouter-alts.json's judge_bias_check_2026_06_15. */
export const DUAL_ROLE_TIP = 'both hosts enabled — try role delegation: claude-flow-codex dual run --template feature|security|refactor (or custom --worker specs)';
export const JUDGE_BIAS_TIP = 'tip: for LLM-judged scoring, use a different vendor than the writer as judge — same-vendor judges run ~8-10pp inflated (still ordinally correct, but not calibrated)';

/** Cross-sell for agentic-qe's qe-court (ADR-124, shipped 3.13.0): its jury
* requires >= 2 distinct vendors seated, which a dual-host setup already
* satisfies. Only meaningful once both hosts are enabled AND aqe is new
* enough to ship the skill — callers gate on both. */
export const QE_COURT_TIP = 'agentic-qe ≥ 3.13.0 ships qe-court (adversarial review) — its jury requires ≥ 2 distinct vendors, which your dual-host setup already satisfies';

/** Suggested aqe-fallback chain when codex is among the enabled hosts: codex's
* models are reached via the `openai` provider type (not as an aqe provider
* itself), so pairing claude-code + openai is a direct inference from the
* hosts already chosen in the same session. Literal reused from
* docs/PROVIDERS.md's own example rather than inventing new model ids. */
export const AQE_FALLBACK_CODEX_SUGGESTION = 'claude-code:claude-opus-4-8; openai:gpt-5.6';
export const suggestedFallbackFor = (enabledHosts) => (enabledHosts.includes('codex') ? AQE_FALLBACK_CODEX_SUGGESTION : null);

// ── agentic-qe router config (.agentic-qe/llm-config.json) ──────────────────
// Grounded in aqe's router config-store + types (ADR-123):
// - mergeRouterConfig deep-merges `providers` but SHALLOW-replaces
Expand Down
58 changes: 58 additions & 0 deletions src/lib/qeCourt.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
// qe-court (ADR-124, agentic-qe >= 3.13.0) — read-only awareness + opt-in
// defaulting of its per-role provider routing, a third configuration surface
// alongside ruflo's host env and aqe's global fallback chain (see issue #36).
//
// vendorOf/validatePanel are ported 1:1 from qe-court's own referee.js (the
// falsifiable, dependency-free core of the court's invariants) so ak can
// report pass/fail without shelling out to aqe. Do not reimplement the court
// protocol itself here — this module only reads/defaults the `routing` block
// of an EXISTING .claude/skills/qe-court/config.json; it never creates the
// file and never touches any other key in it.
import path from 'node:path';
import { readJson } from './settings.mjs';

/** Map a provider id to its coarse vendor — ported from qe-court's referee.js. */
export function vendorOf(providerId) {
const p = String(providerId).toLowerCase();
if (p.startsWith('claude')) return 'claude';
if (p.startsWith('cognitum')) return 'cognitum';
if (p === 'codex' || p === 'openai' || p.startsWith('gpt') || p.startsWith('o3') || p.startsWith('o4')) return 'gpt';
if (p.startsWith('openrouter')) return 'openrouter';
if (p === 'ollama' || p === 'local') return 'local';
return 'unknown';
}

/** Flatten config.json's `routing` map (role -> {provider, model?}) into the
* {role, provider} panel shape validatePanel() expects. */
export function panelFromRouting(routing) {
return Object.entries(routing ?? {})
.filter(([role]) => role !== '_note')
.map(([role, entry]) => ({ role, provider: entry?.provider }));
}

/** Validate a seated panel against the court's anti-collusion invariants.
* Returns a list of violation codes (empty == valid) — ported from
* qe-court's referee.js validatePanel(). */
export function validatePanel(panel, policy = {}) {
const minVendors = policy.minVendors ?? 2;
const violations = [];
const vendorsSeated = new Set(panel.map((s) => vendorOf(s.provider)));
if (vendorsSeated.size < minVendors) violations.push('vendor-diversity');
const jury = panel.find((s) => s.role === 'jury');
const writerLike = panel.filter((s) => s.role === 'defense' || s.role === 'writer');
if (jury) {
const juryVendor = vendorOf(jury.provider);
if (writerLike.some((w) => vendorOf(w.provider) === juryVendor)) violations.push('writerIsNeverJuror');
}
return violations;
}

export function qeCourtConfigPath(root) {
return path.join(root, '.claude', 'skills', 'qe-court', 'config.json');
}

/** Read qe-court's config.json, or null if it hasn't been created yet
* (auto-created by the skill on its first run — ak never creates it). */
export function readQeCourtConfig(root) {
return readJson(qeCourtConfigPath(root), null);
}
65 changes: 65 additions & 0 deletions tests/kit/provider-cli.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
// `ak x provider status` is read-only (detect + report), so — unlike `pick`,
// which can trigger real installs/network calls — it's safe to exercise via a
// real CLI spawn. HOME/XDG_CONFIG_HOME (POSIX) and USERPROFILE/APPDATA
// (Windows) are all pointed at a throwaway sandbox so this never touches the
// real machine's kit.json or ~/.claude — src/lib/paths.mjs's configBase()
// reads APPDATA (not XDG_CONFIG_HOME) on win32, so both must be set or the
// sandbox is silently bypassed there.
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { spawnSync } from 'node:child_process';
import { fileURLToPath } from 'node:url';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { DUAL_ROLE_TIP, JUDGE_BIAS_TIP } from '../../src/lib/providers.mjs';

const BIN = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../bin/agentic-kit.mjs');

function sandbox({ hosts }) {
const home = fs.mkdtempSync(path.join(os.tmpdir(), 'kit-prov-cli-home-'));
const cfgDir = path.join(home, '.config', 'agentic-kit');
fs.mkdirSync(cfgDir, { recursive: true });
fs.writeFileSync(path.join(cfgDir, 'kit.json'), JSON.stringify({ providers: { hosts } }));
const project = fs.mkdtempSync(path.join(os.tmpdir(), 'kit-prov-cli-proj-'));
fs.mkdirSync(path.join(project, '.git'));
return { home, project };
}

function rm(...dirs) {
for (const d of dirs) fs.rmSync(d, { recursive: true, force: true });
}

function ak(args, { cwd, home }) {
const cfgDir = path.join(home, '.config');
return spawnSync(process.execPath, [BIN, ...args], {
encoding: 'utf8',
cwd,
env: {
...process.env,
NO_COLOR: '1',
HOME: home,
USERPROFILE: home, // Windows os.homedir() reads USERPROFILE, not HOME
XDG_CONFIG_HOME: cfgDir,
APPDATA: cfgDir, // Windows configBase() reads APPDATA, not XDG_CONFIG_HOME
},
});
}

test('ak x provider status prints the dual-host guidance tips once both hosts are enabled', () => {
const { home, project } = sandbox({ hosts: { claude: true, codex: true } });
const r = ak(['x', 'provider', 'status'], { cwd: project, home });
assert.equal(r.status, 0, r.stderr);
assert.ok(r.stdout.includes(DUAL_ROLE_TIP), 'dual-role tip printed');
assert.ok(r.stdout.includes(JUDGE_BIAS_TIP), 'judge-bias tip printed');
rm(home, project);
});

test('ak x provider status omits the dual-host guidance tips with only one host enabled', () => {
const { home, project } = sandbox({ hosts: { claude: true, codex: false } });
const r = ak(['x', 'provider', 'status'], { cwd: project, home });
assert.equal(r.status, 0, r.stderr);
assert.ok(!r.stdout.includes(DUAL_ROLE_TIP), 'dual-role tip withheld');
assert.ok(!r.stdout.includes(JUDGE_BIAS_TIP), 'judge-bias tip withheld');
rm(home, project);
});
28 changes: 28 additions & 0 deletions tests/kit/providers.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import {
isDefault, managedEnv, applyHosts, undoProviders, MANAGED_ENV_KEYS,
HOSTS, installHost, applyAqeRouter, undoAqeRouter, aqeRouterFile,
CODEX_ADAPTER_PKG, codexAdapterAction, ensureCodexAdapter, AQE_PROVIDER_TYPES,
bothHostsEnabled, suggestedFallbackFor, AQE_FALLBACK_CODEX_SUGGESTION,
} from '../../src/lib/providers.mjs';

// A tmp dir with a .git marker → settingsTarget() writes the ISOLATED
Expand Down Expand Up @@ -265,6 +266,33 @@ test('ensureCodexAdapter is a no-op without shelling out when codex is disabled'

// ── settingsTarget repo-root walk (scope-leak regression pin) ───────────────

// ── issue #36 Phase 1/2 guidance helpers ────────────────────────────────────

test('bothHostsEnabled is false at the claude-only default', () => {
assert.equal(bothHostsEnabled(defaultCfg()), false);
});

test('bothHostsEnabled is false with only codex enabled', () => {
const cfg = defaultCfg();
cfg.providers.hosts = { claude: false, codex: true };
assert.equal(bothHostsEnabled(cfg), false);
});

test('bothHostsEnabled is true once both hosts are enabled', () => {
const cfg = defaultCfg();
cfg.providers.hosts.codex = true;
assert.equal(bothHostsEnabled(cfg), true);
});

test('suggestedFallbackFor returns the codex-paired suggestion when codex is enabled', () => {
assert.equal(suggestedFallbackFor(['claude', 'codex']), AQE_FALLBACK_CODEX_SUGGESTION);
});

test('suggestedFallbackFor returns null when codex is not among the enabled hosts', () => {
assert.equal(suggestedFallbackFor(['claude']), null);
assert.equal(suggestedFallbackFor([]), null);
});

test('settingsTarget from a repo SUBDIR anchors project scope at the ROOT', async () => {
// A cwd-only .git probe here would fall through to USER scope — leaking
// ENABLE_*/AQE_LLM_PROVIDER machine-wide from any subdir invocation, with
Expand Down
Loading
Loading