diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 585be0c..ec73e7d 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -10,7 +10,7 @@ { "name": "patina", "source": "./", - "version": "6.3.1", + "version": "6.3.2", "description": "Strip the AI packaging, keep the meaning. Detect and rewrite AI writing patterns across KO/EN/ZH/JA with MPS/fidelity checks. Runs the root SKILL.md as the /patina skill.", "homepage": "https://github.com/devswha/patina", "repository": "https://github.com/devswha/patina", diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index bc4baca..faf7b60 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://www.schemastore.org/claude-code-plugin-manifest.json", "name": "patina", - "version": "6.3.1", + "version": "6.3.2", "description": "Detect and rewrite AI writing patterns in Korean, English, Chinese, and Japanese so text reads as if a human wrote it. Meaning-preservation (MPS) verified, audit-friendly. The root SKILL.md is loaded as the /patina skill.", "author": { "name": "devswha", diff --git a/.env.example b/.env.example index c9dd4ec..7ec30d0 100644 --- a/.env.example +++ b/.env.example @@ -1,38 +1,25 @@ -# Local-only runtime / Discord config -# Copy to .env and fill with your own values. - -DISCORD_CHANNEL=your-discord-channel-id -DISCORD_GUILD=your-discord-guild-id -DISCORD_ALLOWED_USERS=comma-separated-user-ids - -PATINA_AGENT_ID=patina -PLANNER_AGENT_ID=planner -GENERATOR_AGENT_ID=generator -EVALUATOR_AGENT_ID=evaluator - -RUNTIME_ENFORCE_ALLOWLIST=true -# ↑ Default flipped to true. Bootstrap will refuse to start unless -# DISCORD_ALLOWED_USERS is set. Set this back to false ONLY if you understand -# that any user who can post in the bound channel can drive the runtime. -PATINA_RUNTIME_CLI=your-runtime-cli-binary - -# Optional: override if your old bot token lives elsewhere -# CLAWHIP_CONFIG=/home/you/.clawhip/config.toml -# Optional: direct token override if your runtime does not reuse the migrated bot token -# RUNTIME_DISCORD_TOKEN= - -# Optional: isolated marketing bot profile (separate OpenClaw state/config) -# MARKETING_RUNTIME_PROFILE=marketing -# MARKETING_AGENT_ID=patina-marketing -# MARKETING_SOURCE_AGENT_ID=patina -# MARKETING_DISCORD_GUILD=your-discord-guild-id -# MARKETING_DISCORD_CHANNEL=your-marketing-channel-id -# MARKETING_DISCORD_ALLOWED_USERS=comma-separated-user-ids -# MARKETING_DISCORD_TOKEN= -# MARKETING_GATEWAY_PORT=18889 -# MARKETING_WORKSPACE=/home/you/.openclaw-marketing/workspace -# MARKETING_ENFORCE_ALLOWLIST=true # default; bootstrap refuses without MARKETING_DISCORD_ALLOWED_USERS -# MARKETING_RESTART_GATEWAY=false +# patina environment reference — copy to .env and fill only what you use. +# .env is gitignored; never commit real values. + +# --------------------------------------------------------------------------- +# CLI / local LLM provider keys (openai-http backend, quality:live harness) +# Lookup order: PATINA_API_KEY first, then the provider-specific key +# (src/auth.js HTTP_KEY_ENV_VARS). `--provider ` presets live in +# src/providers.js: openai, gemini, groq, kimi, moonshot, together. +# --------------------------------------------------------------------------- +# PATINA_API_KEY= # generic key, wins over provider-specific vars +# OPENAI_API_KEY= # default openai-http backend (gpt-5.5) +# GEMINI_API_KEY= # --provider gemini (gemini-2.5-pro) +# GROQ_API_KEY= # --provider groq +# TOGETHER_API_KEY= # --provider together +# KIMI_API_KEY= # --provider kimi (kimi-k2.5, api.moonshot.ai) +# MOONSHOT_API_KEY= # --provider moonshot (same endpoint as kimi) +# PATINA_API_KEY_FILE= # or read any key from a file (recommended for CI) + +# Local subscription CLIs (claude / codex / gemini / kimi) need no key here — +# a logged-in seat is the credential. See also PATINA_LIVE_* and +# PATINA_LIVE_JUDGE_* (fixed-judge scoring) in tests/quality/README.md. + # --------------------------------------------------------------------------- # Hosted playground / API service (Vercel `/api/rewrite`) # Server-side only. The browser never sees these. All tiers fail closed: @@ -46,9 +33,15 @@ PATINA_RUNTIME_CLI=your-runtime-cli-binary # Free tier: the service's own provider key (rate-limited by IP) + HMAC secret # that signs quota/subject KV keys (raw keys/licenses are never stored). +# +# Model choice (owner-approved 2026-07-26): gemini-3.6-flash. The prior default +# gpt-4.1-mini measured an AI-score improvement of only 0.2 across 22 fixtures +# — it returns the input nearly unchanged, so free traffic received no real +# rewrite while still scoring high on meaning preservation. gemini-3.6-flash +# measured 13.0. See docs/operations/serving-engine-cost-20260725.md. # PATINA_FREE_API_KEY=your-provider-key -# PATINA_FREE_PROVIDER=openai -# PATINA_FREE_MODEL=gpt-4.1-mini +# PATINA_FREE_PROVIDER=gemini +# PATINA_FREE_MODEL=gemini-3.6-flash # PATINA_QUOTA_HMAC_SECRET=your-long-random-secret # --- Pro tier ($9.99/mo USD, Lemon Squeezy license-gated) ------------------- @@ -72,10 +65,42 @@ PATINA_RUNTIME_CLI=your-runtime-cli-binary # Pro provider/model — REQUIRED in production (missing values fail pro requests # closed with 503; no silent fallback to the free provider/model). Outside # production they fall back to PATINA_FREE_PROVIDER/MODEL, then the preset. -# Both must be on the PROVIDER_PRESETS allowlist. Pro ships on Claude Sonnet 5. +# Both must be on the PROVIDER_PRESETS allowlist. +# +# Model choice (owner-approved 2026-07-26; evidence re-established 2026-07-27): +# gemini-3.6-flash, replacing claude-sonnet-5. Rerun on 22 live-quality +# fixtures after fixing a broken fidelity rubric and a harness prompt that +# omitted the persona (docs/operations/serving-engine-cost-20260725.md): the +# two engines both score 20/22 with MPS 90.4 vs 91.5. Quality is level; the +# case is cost and latency — $0.030 vs $0.156 per rewrite, 8.3s vs 27.7s. +# PATINA_PRO_PROVIDER=gemini +# PATINA_PRO_MODEL=gemini-3.6-flash +# +# Previous pin, kept for rollback: # PATINA_PRO_PROVIDER=claude # PATINA_PRO_MODEL=claude-sonnet-5 +# --- Prompt caching (Anthropic only, opt-in) -------------------------------- +# The rewrite prompt is ~99.7% fixed prefix (pattern catalog + profile + voice) +# with only the user text after the data fence, so it caches almost perfectly. +# Anthropic's OpenAI-compat endpoint ignores cache_control, so this flag routes +# paid calls through the native /v1/messages adapter with a cached prefix. +# +# Measured 2026-07-25 (ko web prompt, claude-sonnet-5): 34,254 cacheable prefix +# tokens; second identical call read 34,254/34,254 from cache and halved latency +# (25.2s -> 12.0s). Cost per request: $0.126 uncached, $0.151 on a cache miss +# (+20%, the 5m write surcharge), $0.033 on a hit (-74%). +# +# Break-even hit rate is 21.7%, i.e. roughly 3 requests/hour per language +# variant (the 5-minute ephemeral TTL refreshes on each hit). Below that this +# flag is a ~20% input surcharge; above it, savings approach 74%. +# PATINA_ANTHROPIC_NATIVE_CACHE=1 +# +# Thinking stays at the provider default (ON for sonnet-5) because a measured +# A/B showed thinking-off rewrites amputate content. Set to 0 only for +# experiments, never to cut cost on the paid path. +# PATINA_ANTHROPIC_THINKING=1 + # Pro limits (per-license). Defaults shown. # PATINA_PRO_MAX_CHARS=20000 # PATINA_PRO_REQ_PER_DAY=200 diff --git a/.patina.default.yaml b/.patina.default.yaml index abc7ef1..3dfaa6e 100644 --- a/.patina.default.yaml +++ b/.patina.default.yaml @@ -1,4 +1,4 @@ -version: "6.3.1" +version: "6.3.2" language: ko # Korean (default) -- auto-loads all ko-*.md patterns # language: en -- English -- auto-loads all en-*.md patterns # language: zh -- Chinese -- auto-loads all zh-*.md patterns diff --git a/CHANGELOG.md b/CHANGELOG.md index 133c39f..ae15f4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,29 @@ All notable changes to patina. Dates are release dates (YYYY-MM-DD). Semver rationale: patch | minor | major — explain whether this changes patterns, schemas, CLI behavior, or docs only. ``` +## 6.3.2 — 2026-07-29 + +**Hosted-rewrite fixes (chat-log pastes no longer 422), a fidelity-gate scoring fix, backend compatibility, and opt-in LLM plumbing. 6.4 stays reserved for the payment launch.** + +Semver rationale: patch — bug fixes on the hosted rewrite surface, scoring, and CLI backends, plus strictly opt-in or allowlist-only additions; no default contract, tier limit, or pinned model changes. The v6.4 payment-launch hold (tag/publish prohibitions for 6.4.x) is untouched. + +### Fixed + +- **Chat-log/timeline pastes no longer fail the web rewrite**: clock times (`16:47`, `16:47 – 16:50`, `9:05:30`) hit the numeric-operator check as `digit:digit`, so the number-safety gate 422-rejected the whole paste before scoring — even for an identity rewrite — and the playground showed a misleading generic "check the mode/key" failure. Clock times are now first-class exact `time:` claims (leading-zero hour normalized); a rewrite that drops or drifts a time still fails closed as `numeric_claim_changed`, and ratios/scores (`1:2`, `3:1`) plus invalid times (`25:30`) keep the old fail-closed behavior. `number_safety_failed` also gets its own localized (en/ko/zh/ja) error copy in the playground. +- **Fidelity gate no longer punishes the rewrite it exists to grade**: the fidelity judge rubric penalized register normalization that the rewrite prompt itself mandates; live-quality fixtures went from 9/22 to 20/22 after the rubric fix plus prompt parity between the CLI and web surfaces. +- **Rewrite prompt forbids invented claims**: the strict rewrite prompt now explicitly prohibits adding claims the source does not state, closing a fabrication path the meaning floors could only catch after the fact. +- **`gemini-cli` backend disables MCP servers** on its invocations, so a user's local MCP config can never inject tools into a patina rewrite call. +- **`kimi-cli` ≥ 0.28 compatibility**: the backend now parses the stream-json assistant output of newer Kimi Code releases (with the legacy plain-text fallback retained). +- **SSE streams request usage accounting** (`include_usage`), so streamed rewrites report token usage like buffered calls. +- **Playground approval status line is screen-reader-only**: the visible "Unapproved — checks have not passed" line under every streaming message read as an alarming warning during normal in-flight rewrites; it is now visually clipped while the `role="status"` live region and localized copy stay intact for assistive tech. + +### Added + +- **`gemini-3.6-flash` allowlisted** on the web BYOK surface and eligible for `PATINA_FREE_MODEL` / `PATINA_PRO_MODEL`, after a 22-fixture live-quality comparison (better AI-score improvement and fewer meaning-loss fixtures than the prior Pro pin at ~1/5 the cost). All pinned defaults are unchanged; this only widens the allowlist. +- **Native Anthropic adapter with prompt caching (opt-in)** plus thinking control for OpenAI-compatible providers via `extraBody` pass-through — both dormant unless explicitly enabled. +- **Free-tier observability**: the pro monitor now also watches the free tier (the tier that actually has users), with the same closed outcome schema. +- **Live-quality harness upgrades** (dev tooling): fixed-judge override, `--judge-backend` / `--backend` subscription-CLI seats, per-call usage/latency capture, and `--repeat` for variance-aware sweeps. + ## 6.3.1 — 2026-07-07 **Launch polish for the hosted playground and README — no CLI/engine changes.** diff --git a/README.md b/README.md index 1538427..2b2ffc5 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ License: MIT Skill: Claude Code | Codex | Cursor | OpenCode Languages: KO | EN | ZH | JA - Version 6.3.1 + Version 6.3.2

@@ -204,7 +204,7 @@ If meaning drifts, the change is retried or rolled back. Deterministic analysis ```yaml # .patina.default.yaml -version: "6.3.1" +version: "6.3.2" language: ko # ko | en | zh | ja profile: default output: rewrite # rewrite | diff | audit | score diff --git a/README_JA.md b/README_JA.md index 130f14f..b7142a8 100644 --- a/README_JA.md +++ b/README_JA.md @@ -6,7 +6,7 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![Skill](https://img.shields.io/badge/Skill-Claude%20Code%20%7C%20Codex%20%7C%20Cursor%20%7C%20OpenCode-blueviolet)](#クイックスタート) [![Multi-language](https://img.shields.io/badge/Languages-KO%20%7C%20EN%20%7C%20ZH%20%7C%20JA-green)](https://github.com/devswha/patina) -[![Version](https://img.shields.io/badge/version-6.3.1-blue)](CHANGELOG.md) +[![Version](https://img.shields.io/badge/version-6.3.2-blue)](CHANGELOG.md) > **AIっぽさだけを落として、意味はそのまま。** @@ -186,7 +186,7 @@ Input ```yaml # .patina.default.yaml -version: "6.3.1" +version: "6.3.2" language: ko # ko | en | zh | ja profile: default output: rewrite # rewrite | diff | audit | score diff --git a/README_KR.md b/README_KR.md index 72af351..cab3034 100644 --- a/README_KR.md +++ b/README_KR.md @@ -6,7 +6,7 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![Skill](https://img.shields.io/badge/Skill-Claude%20Code%20%7C%20Codex%20%7C%20Cursor%20%7C%20OpenCode-blueviolet)](#빠른-시작) [![Multi-language](https://img.shields.io/badge/Languages-KO%20%7C%20EN%20%7C%20ZH%20%7C%20JA-green)](https://github.com/devswha/patina) -[![Version](https://img.shields.io/badge/version-6.3.1-blue)](CHANGELOG.md) +[![Version](https://img.shields.io/badge/version-6.3.2-blue)](CHANGELOG.md)

브라우저에서 바로 써보기 — 설치 없음 @@ -186,7 +186,7 @@ Input ```yaml # .patina.default.yaml -version: "6.3.1" +version: "6.3.2" language: ko # ko | en | zh | ja profile: default output: rewrite # rewrite | diff | audit | score diff --git a/README_ZH.md b/README_ZH.md index 9661627..7cd27d8 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -6,7 +6,7 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![Skill](https://img.shields.io/badge/Skill-Claude%20Code%20%7C%20Codex%20%7C%20Cursor%20%7C%20OpenCode-blueviolet)](#快速开始) [![Multi-language](https://img.shields.io/badge/Languages-KO%20%7C%20EN%20%7C%20ZH%20%7C%20JA-green)](https://github.com/devswha/patina) -[![Version](https://img.shields.io/badge/version-6.3.1-blue)](CHANGELOG.md) +[![Version](https://img.shields.io/badge/version-6.3.2-blue)](CHANGELOG.md)

去掉 AI 味，保留原意。 @@ -188,7 +188,7 @@ Input ```yaml # .patina.default.yaml -version: "6.3.1" +version: "6.3.2" language: ko # ko | en | zh | ja profile: default output: rewrite # rewrite | diff | audit | score diff --git a/SKILL.md b/SKILL.md index 5488f36..1aec95f 100644 --- a/SKILL.md +++ b/SKILL.md @@ -1,6 +1,6 @@ --- name: patina -version: 6.3.1 +version: "6.3.2" description: Detect and rewrite AI writing patterns in Korean, English, Chinese, and Japanese text so it reads as if a human wrote it. Meaning-preservation (MPS) verified. allowed-tools: - Read diff --git a/api/pro-monitor.js b/api/pro-monitor.js index fddf002..a8afcca 100644 --- a/api/pro-monitor.js +++ b/api/pro-monitor.js @@ -1,7 +1,7 @@ // @ts-check import { createHash } from 'node:crypto'; import { TextDecoder } from 'node:util'; -import { evaluateProMonitor, isCronAuthorized, SYNTHETIC_TEXT, LATENCY_BUCKETS } from '../src/pro-monitor.js'; +import { evaluateFreeTierHealth, evaluateProMonitor, isCronAuthorized, SYNTHETIC_TEXT, LATENCY_BUCKETS } from '../src/pro-monitor.js'; import { WEB_OBSERVABILITY_SCHEMA } from '../src/web-observability.js'; const DEADLINE_MS = 55_000; @@ -62,6 +62,11 @@ function parseAggregate(payload, window) { const value = payload?.data && typeof function createLogQuery(env, fetchImpl, start) { const endpoint = publicServiceUrl(env.PATINA_VERCEL_LOG_QUERY_URL); const token = env.PATINA_VERCEL_LOG_QUERY_TOKEN; if (!endpoint || !required(token) || !/^[a-f0-9]{64}$/.test(env.PATINA_VERCEL_LOG_QUERY_URL_SHA256 || '') || textHash(endpoint.toString()) !== env.PATINA_VERCEL_LOG_QUERY_URL_SHA256) return null; return async ({ channel, tier, window, aggregateOnly, readOnly }) => { if (!['production', 'staging'].includes(channel) || tier !== 'pro' || !['15m', '30m'].includes(window) || !aggregateOnly || !readOnly) throw new Error('scope'); const url = new URL(endpoint); url.searchParams.set('channel', channel); url.searchParams.set('tier', tier); url.searchParams.set('window', window); url.searchParams.set('aggregate_only', 'true'); const response = await fetchBounded(fetchImpl, start, url, { method: 'GET', redirect: 'error', headers: { Authorization: `Bearer ${token}`, Accept: 'application/json' } }); if (!response.ok) throw new Error('logs'); return parseAggregate(await bodyJson(response, start), window); }; } function parseSynthetic(text) { const lines = text.split(/\r?\n/).filter(Boolean); let started = false; let done = null; for (const line of lines) { let frame; try { frame = JSON.parse(line); } catch { return false; } if (!frame || typeof frame !== 'object' || Array.isArray(frame) || done) return false; if (frame.type === 'start' && !started && Object.keys(frame).length === 1) started = true; else if (frame.type === 'delta' && started && typeof frame.text === 'string' && Object.keys(frame).every((key) => key === 'type' || key === 'text')) { continue; } else if (frame.type === 'done' && started && typeof frame.rewrite === 'string' && frame.rewrite.length > 0) done = frame; else return false; } return Boolean(started && done); } function createSynthetic(env, fetchImpl, start) { const origin = publicServiceUrl(env.PATINA_PUBLIC_BASE_URL); if (!origin || origin.pathname !== '/' || !required(env.PATINA_PUBLIC_BASE_URL_SHA256) || !/^[a-f0-9]{64}$/.test(env.PATINA_PUBLIC_BASE_URL_SHA256) || textHash(origin.toString()) !== env.PATINA_PUBLIC_BASE_URL_SHA256 || !required(env.PATINA_SYNTHETIC_PRO_LICENSE) || !required(env.PATINA_SYNTHETIC_OBSERVER_SECRET)) return null; return async () => { try { const response = await fetchBounded(fetchImpl, start, new URL('/api/rewrite', origin), { method: 'POST', redirect: 'error', headers: { Authorization: `Bearer ${env.PATINA_SYNTHETIC_PRO_LICENSE}`, 'Content-Type': 'application/json', Accept: 'application/x-ndjson', 'x-patina-synthetic-observer': env.PATINA_SYNTHETIC_OBSERVER_SECRET }, body: JSON.stringify({ mode: 'first', lang: 'en', tier: 'pro', text: SYNTHETIC_TEXT }) }); return response.ok && /^application\/x-ndjson(?:;|\s|$)/i.test(response.headers?.get?.('content-type') ?? '') && parseSynthetic(await bodyText(response, start)) ? { ok: true, terminal: 'done' } : { ok: false, terminal: 'failed' }; } catch { return { ok: false, terminal: 'failed' }; } }; } +// Same probe as the pro synthetic, minus the license, so it exercises the tier +// real users are on. The observer header keeps the probe out of the aggregate +// it watches; it does NOT exempt the request from the free IP quota, which is +// why evaluateFreeTierHealth budgets how often this may run. +function createFreeCanary(env, fetchImpl, start) { const origin = publicServiceUrl(env.PATINA_PUBLIC_BASE_URL); if (!origin || origin.pathname !== '/' || !required(env.PATINA_PUBLIC_BASE_URL_SHA256) || !/^[a-f0-9]{64}$/.test(env.PATINA_PUBLIC_BASE_URL_SHA256) || textHash(origin.toString()) !== env.PATINA_PUBLIC_BASE_URL_SHA256 || !required(env.PATINA_SYNTHETIC_OBSERVER_SECRET)) return null; return async () => { try { const response = await fetchBounded(fetchImpl, start, new URL('/api/rewrite', origin), { method: 'POST', redirect: 'error', headers: { 'Content-Type': 'application/json', Accept: 'application/x-ndjson', 'x-patina-synthetic-observer': env.PATINA_SYNTHETIC_OBSERVER_SECRET }, body: JSON.stringify({ mode: 'first', lang: 'en', tier: 'free', text: SYNTHETIC_TEXT }) }); return response.ok && /^application\/x-ndjson(?:;|\s|$)/i.test(response.headers?.get?.('content-type') ?? '') && parseSynthetic(await bodyText(response, start)) ? { ok: true, terminal: 'done' } : { ok: false, terminal: 'failed' }; } catch { return { ok: false, terminal: 'failed' }; } }; } function createDiscord(env, fetchImpl, start) { const url = safeHttps(env.PATINA_ALERT_DISCORD_WEBHOOK, (host) => DISCORD_HOSTS.has(host)); if (!url || !/^\/api\/webhooks\/\d+\/[A-Za-z0-9._-]+$/.test(url.pathname)) return null; url.searchParams.set('wait', 'true'); return async (payload) => { const response = await fetchBounded(fetchImpl, start, url, { method: 'POST', redirect: 'error', headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, body: JSON.stringify(payload) }); if (!response.ok) return { status: response.status }; try { const data = await bodyJson(response, start); return { status: response.status, receiptId: SAFE_ID.test(data?.id || '') ? data.id : undefined }; } catch { return { status: response.status }; } }; } function int(value) { return Number.isSafeInteger(value) && value >= 0 ? value : null; } function plain(value) { return value !== null && typeof value === 'object' && !Array.isArray(value) && (Object.getPrototypeOf(value) === Object.prototype || Object.getPrototypeOf(value) === null); } @@ -80,11 +85,14 @@ function validPending(env, item, id, channel) { return exact(item, PENDING_KEYS) function band(count) { return count === 1 ? '1' : count < 5 ? '2-4' : count < 10 ? '5-9' : count < 20 ? '10-19' : '20+'; } function receipt(item, recovery) { const payload = { schemaVersion: 'OBS-ALERT-v1', receiptId: item.receiptId, issuedAt: recovery.issuedAt, issuer: 'patina.pro-monitor', deploymentId: item.deploymentId, channel: item.channel, tier: 'pro', realPath: true, namespace: NAMESPACE, eventSchema: item.eventSchema, eventSchemaVersion: item.eventSchemaVersion, eventSchemaHash: item.eventSchemaHash, configHash: item.configHash, ruleVersion: item.ruleVersion, trigger: item.trigger, window: item.triggerFact.window, countBand: band(item.triggerFact.count), denominators: item.denominators, latency: { counts: item.histogram.counts, n: item.histogram.n, p95Rank: item.histogram.rank, over120Ratio: item.histogram.over120Ratio, ruleVersion: item.ruleVersion }, cronAuthorized: true, syntheticTerminal: item.syntheticTerminal, syntheticStreak: item.syntheticStreak, discord: { status: '2xx', attempts: item.alert.attempts }, dedupControlKey: `${CONTROL}:${item.channel}:pro:dedup:${item.trigger}`, pendingAlertKey: `${CONTROL}:${item.channel}:pro:pending:${item.receiptId}`, recoveryId: recovery.receiptId }; const final = { ...payload, artifactHash: hash(payload) }; if (!exact(final, FINAL_KEYS)) throw new Error('receipt'); return final; } function summary(value) { return { channel: value.channel, tier: 'pro', windows: value.buckets?.length || 0, histogram: value.histogram, syntheticStreak: value.syntheticStreak, signals: value.triggers?.map((x) => x.trigger) || [], alerts: value.alerts?.map(({ trigger, sent, deduped }) => ({ trigger, sent: sent === true, deduped: deduped === true })) || [] }; } -export function createProMonitorApiHandler({ env = process.env, fetchImpl = globalThis.fetch, evaluateProMonitorImpl = evaluateProMonitor } = {}) { +// A free-monitor result is reported, never fatal: the cron's status code +// speaks for the pro evidence run only. +function freeSummary(value) { if (!value || value.error) return { available: false }; return { available: true, tier: value.tier, total: value.denominators?.total ?? 0, failed: value.denominators?.failed ?? 0, canary: value.canaryTerminal, signals: value.triggers?.map((x) => x.trigger) ?? [] }; } +export function createProMonitorApiHandler({ env = process.env, fetchImpl = globalThis.fetch, evaluateProMonitorImpl = evaluateProMonitor, evaluateFreeTierHealthImpl = evaluateFreeTierHealth } = {}) { return async (req, res) => { if (req?.method !== 'GET' || !empty(req?.body)) return send(res, 405, { error: 'method_not_allowed' }); if (!authorized(req, env.CRON_SECRET)) return send(res, 401, { error: 'unauthorized' }); - const start = Date.now(); const kv = createKv(env, fetchImpl, start); const logs = createLogQuery(env, fetchImpl, start); const synthetic = createSynthetic(env, fetchImpl, start); const discord = createDiscord(env, fetchImpl, start); + const start = Date.now(); const kv = createKv(env, fetchImpl, start); const logs = createLogQuery(env, fetchImpl, start); const synthetic = createSynthetic(env, fetchImpl, start); const freeCanary = createFreeCanary(env, fetchImpl, start); const discord = createDiscord(env, fetchImpl, start); if (!['production', 'staging'].includes(env.PATINA_DEPLOYMENT_CHANNEL) || !DEPLOYMENT_ID.test(env.VERCEL_GIT_COMMIT_SHA || '') || !kv || !logs || !synthetic || !discord) return send(res, 503, { error: 'monitor_unavailable' }); try { const sleep = async (ms) => { if (ms > deadline(start)) throw new Error('deadline'); await race(new Promise((resolve) => setTimeout(resolve, ms)), deadline(start)); }; @@ -101,9 +109,17 @@ export function createProMonitorApiHandler({ env = process.env, fetchImpl = glob return { channel: fact.channel, originals, finals, recovery }; }; const value = await race(evaluateProMonitorImpl({ channel: env.PATINA_DEPLOYMENT_CHANNEL, tier: 'pro', aggregateReader: kv, controlStore: kv, logQuery: logs, syntheticRequest: synthetic, discordSender: discord, prepareAlertEvidence, prepareRecoveryEvidence, sleep, deadlineMs: Math.min(30_000, deadline(start)) }), deadline(start)); + // The free tier is where the users are while checkout is disabled, and + // on 2026-07-27 it failed three times without alerting because nothing + // read its counters. Evaluated after the paid path and never allowed to + // fail the cron: a canary problem must not mask the pro evidence run. + let free = null; + try { + free = await race(evaluateFreeTierHealthImpl({ channel: /** @type {'production'|'staging'} */ (env.PATINA_DEPLOYMENT_CHANNEL), tier: 'free', aggregateReader: kv, controlStore: kv, canaryRequest: freeCanary ?? undefined, discordSender: discord, sleep, deadlineMs: Math.min(10_000, deadline(start)) }), deadline(start)); + } catch { free = { error: 'free_monitor_unavailable' }; } const blindUnacked = value.alerts?.some((item) => item.trigger === 'monitor_blind' && !item.deduped && !item.sent); if (!value.adapters?.aggregate || !value.adapters?.safetyEntitlementLogs || !value.adapters?.monitorDropLogs || blindUnacked) return send(res, 503, { error: 'monitor_unavailable' }); - return send(res, 200, summary(value)); + return send(res, 200, { ...summary(value), free: freeSummary(free) }); } catch { return send(res, 503, { error: 'monitor_unavailable' }); } }; } diff --git a/core/scoring.md b/core/scoring.md index 786fe1f..4409870 100644 --- a/core/scoring.md +++ b/core/scoring.md @@ -288,6 +288,10 @@ Four criteria, each scored independently by the LLM comparing original → outpu ### 10.1 Claims Preserved Every factual claim in the original appears (perhaps rephrased) in the output. +Stylistic packaging is not a claim. Hype and intensifiers ("cutting-edge", +"unprecedented", "seamlessly") carry no checkable content, so removing them is +the rewrite's job and is never scored as loss — the same exemption the +Meaning-Preservation Score applies. | Level | Points | Criteria | |-------|--------|----------| @@ -309,11 +313,14 @@ The output does not add claims, facts, or specifics not present or implied by th ### 10.3 Tone Match -The output's register matches the original (or the profile's target register, if explicitly overridden). +The output serves the same audience and domain register as the original (or the +profile's target register, if explicitly overridden). This judges function, not +surface polish: stripping AI-ish formality, hype, or ceremony while the audience +and domain hold is High, not drift. | Level | Points | Criteria | |-------|--------|----------| -| High | 3 | Tone matches — formality level, domain register, and audience consistent | +| High | 3 | Same audience and domain — a policy notice still reads as a policy notice, a product page as a product page | | Medium | 2 | Slight drift — somewhat more/less formal, but still appropriate for the context | | Low | 1 | Noticeable mismatch — formal original made casual (or vice versa) without profile justification | | Fail | 0 | Register violation — academic text made into slang, or casual text made into legalese | @@ -327,10 +334,10 @@ Compares output length to original. Extreme changes suggest content loss or padd | Ratio | Points | Criteria | |-------|--------|----------| -| 70–130% | 3 | Length preserved — natural variation within ±30% | -| 50–69% or 131–150% | 2 | Moderate change — some compression or expansion, likely acceptable | -| 30–49% or 151–200% | 1 | Significant change — substantial content probably lost or padded | -| < 30% or > 200% | 0 | Extreme change — content almost certainly lost or heavily padded | +| 50–130% | 3 | Length preserved, or compressed as expected when packaging is stripped | +| 35–49% or 131–150% | 2 | Moderate change — heavy compression or expansion, likely acceptable | +| 25–34% or 151–200% | 1 | Significant change — substantial content probably lost or padded | +| < 25% or > 200% | 0 | Extreme change — content almost certainly lost or heavily padded | **Calculation:** `length_ratio = len(output) / len(original) × 100` @@ -353,7 +360,8 @@ To reduce variance, apply these guidelines when scoring fidelity criteria: - Inventing a statistic, date, or name not in the original → Low or Fail. ### Tone Match -- Compare the first and last paragraphs of original vs. output for register cues. +- Compare the first and last paragraphs of original vs. output for audience and domain cues. +- Removing hype, ceremony, or AI-ish formality is the rewrite's job → High. - Profile-targeted register shifts are expected, not penalized. - Mixed register (formal opening, casual middle) counts as Low. @@ -407,12 +415,12 @@ Output: Humanized version. |-----------|-------|-----------| | Claims preserved | 3 (High) | All policy recommendations and cited figures present | | No fabrication | 2 (Medium) | Added "as widely reported" — minor inference stated as fact | -| Tone match | 3 (High) | Academic register maintained throughout | -| Length ratio | 2 (Moderate) | Output is 68% of original length (within 50-69% band) | +| Tone match | 3 (High) | Academic audience and domain held throughout | +| Length ratio | 3 (High) | Output is 68% of original length (within the 50–130% band) | -Fidelity = (3+2+3+2) / 12 × 100 = **83.3** +Fidelity = (3+2+3+3) / 12 × 100 = **91.7** -Interpretation: 76-90 range = "높은 충실도" (High fidelity, minor issues) +Interpretation: 91-100 range = "완벽한 충실도" (Perfect fidelity, all meaning preserved) ### Fidelity Score Interpretation diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index d914a83..245d316 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -310,6 +310,36 @@ Campaign state: The entitlement layer stays provider-agnostic (injected license validator, env-driven checkout URL) so an adapter remains bounded work; do not build it speculatively before a trigger fires. +- Serving engine (2026-07-26, owner-approved; evidence re-established + 2026-07-27): both tiers run `gemini-3.6-flash`. The original comparison was + taken under a broken fidelity rubric and a harness prompt missing the + persona, so it was void. Rerun with both fixed, on subscription seats: + gemini-3.6-flash and claude-sonnet-5 both score **20/22** (ko 11/11, + en 9/11), MPS 90.4 vs 91.5, fidelity 92.4 vs 94.7. Quality is level; the case + for the swap is cost and latency — $0.030 vs $0.156 per rewrite, 8.3s vs + 27.7s. Evidence: `docs/operations/serving-engine-cost-20260725.md`. +- Cheaper engines were rerun on the fixed apparatus and none can undercut the + shipped one. `deepseek-v4-flash` ($0.003) rewrites hard but gutted + `ko-news-01` to MPS 24; `gemini-3.5-flash-lite` ($0.007) preserves meaning + and barely rewrites — 8 of 22 fixtures carry `ai_not_improved`, the same + evasion that made `gpt-4.1-mini` look like a leader. $0.030 per rewrite is + the current floor, not a number to shave. The OpenAI-hosted candidates remain + unmeasured on an exhausted account balance; none is in production. +- Register failures (opened 2026-07-26, **closed 2026-07-27**): five registers + appeared to fail on every engine across a 20x price spread. Both causes were + in the measuring apparatus, not the engines. The fidelity rubric charged + removal of the stylistic packaging patina exists to strip as omitted claims — + production was returning `floor_failed` to real users for correct rewrites — + and the live-quality harness built its prompt without the persona, so it + measured meaning preservation on rewrites that were never told to preserve + meaning. Fixing both took the same engine on the same 22 fixtures from 9/22 + to **20/22**, and Korean from 2/11 to **11/11**. Evidence: + `docs/operations/register-failure-handoff-20260726.md`. +- **Open**: two English fixtures, failing in opposite directions. + `en-marketing-01` strips hype thoroughly (AI 35.6 → 5.7) but drops anchors + (MPS 60); `en-public-docs-01` preserves meaning and barely improves the AI + score (15.6 → 16.5). Add a second fixture per register before treating either + as a register-wide pattern. Next recommended order: diff --git a/docs/benchmarks/detector-comparison.json b/docs/benchmarks/detector-comparison.json index 260952f..bd47ab1 100644 --- a/docs/benchmarks/detector-comparison.json +++ b/docs/benchmarks/detector-comparison.json @@ -1,8 +1,8 @@ { "reportVersion": 1, - "generatedAt": "2026-07-02T06:03:59.439Z", + "generatedAt": "2026-07-29T08:50:08.370Z", "fixtureCount": 49, - "benchmarkGeneratedAt": "2026-07-02T06:03:59.425Z", + "benchmarkGeneratedAt": "2026-07-29T08:50:08.356Z", "note": "Offline comparison protocol. Built-in Patina row uses deterministic suspect-zone analyzer; third-party rows are manual opt-in only.", "manualInput": null, "detectors": [ diff --git a/docs/benchmarks/detector-comparison.md b/docs/benchmarks/detector-comparison.md index 84e68df..b340908 100644 --- a/docs/benchmarks/detector-comparison.md +++ b/docs/benchmarks/detector-comparison.md @@ -4,7 +4,7 @@ This report is generated offline from the checked-in suspect-zone fixtures. It i ## Current run -- Generated at: 2026-07-02T06:03:59.439Z +- Generated at: 2026-07-29T08:50:08.370Z - Fixture source: `tests/fixtures/suspect-zones/**` - Fixture count: 49 - Manual third-party input: none diff --git a/docs/benchmarks/latest.json b/docs/benchmarks/latest.json index ac9d687..fa63f7b 100644 --- a/docs/benchmarks/latest.json +++ b/docs/benchmarks/latest.json @@ -6,8 +6,8 @@ "regressionRanges": "tests/fixtures/suspect-zones/expected-ranges.json", "schemaVersion": 3, "fixtureSchemaVersion": 1, - "nodeVersion": "v22.17.1", - "generatedAt": "2026-07-02T06:03:59.188Z", + "nodeVersion": "v24.18.0", + "generatedAt": "2026-07-29T08:50:08.132Z", "fixtureCount": 49, "overallAccuracy": 1, "overall": { diff --git a/docs/benchmarks/latest.md b/docs/benchmarks/latest.md index dcc0b93..701f2a4 100644 --- a/docs/benchmarks/latest.md +++ b/docs/benchmarks/latest.md @@ -7,8 +7,8 @@ This is the latest checked-in report for patina's deterministic suspect-zone ben ## Current result - Status: **passing** -- Generated at: 2026-07-02T06:03:59.188Z -- Node: v22.17.1 +- Generated at: 2026-07-29T08:50:08.132Z +- Node: v24.18.0 - Fixture schema: v1 - Fixtures: 49 - Languages: 4 (en, ja, ko, zh) diff --git a/docs/operations/pay-b-cost-20260724-cached.json.bundle.json b/docs/operations/pay-b-cost-20260724-cached.json.bundle.json new file mode 100644 index 0000000..5f284b2 --- /dev/null +++ b/docs/operations/pay-b-cost-20260724-cached.json.bundle.json @@ -0,0 +1 @@ +{"financialInputs":{"bootstrap":{"confidenceBps":9500,"iterations":10000,"seed":"PAY-B-20260723-1236551-1932893"},"feeUsdMicros":999500,"refundReserveUsdMicros":499500,"unitChars":1000000},"pricing":{"cacheCreationBillingGranularityTokens":1,"cacheCreationUsdMicrosPerMillionTokens":3750000,"cacheReadBillingGranularityTokens":1,"cacheReadUsdMicrosPerMillionTokens":300000,"inputBillingGranularityTokens":1,"inputUsdMicrosPerMillionTokens":3000000,"minimumChargeUsdMicros":1,"outputBillingGranularityTokens":1,"outputUsdMicrosPerMillionTokens":15000000,"source":"https://platform.claude.com/docs/en/about-claude/pricing","sourceChars":"Claude Sonnet 5 starting September 1, 2026: Base Input $3 / MTok; 5m Cache Writes $3.75 / MTok; 1h Cache Writes $6 / MTok; Cache Hits & Refreshes $0.30 / MTok; Output $15 / MTok. Introductory pricing of $2/$10 per million input/output tokens is in effect through August 31, 2026, after which the standard pricing of $3/$15 per million input/output tokens will take effect. Retrieved 2026-07-24 from https://platform.claude.com/docs/en/about-claude/pricing (model pricing table). Standard post-introductory rates are used as the base for margin evaluation.","sourceSha256":"8bafd83ba9b47388be151750679a349be763fb74b45671c0740678ade4d80a59"},"providerBillingFacts":[{"attemptIndex":1,"billingEvidence":{"billed":true,"externalReferenceSha256":"c76966c64dd5d9aeb1949e470315e45689b0a402c99248216d2ed7d1893a42a2","provider":"claude","providerReportedAmountUsdMicros":null,"rawUsageSha256":"ae794d044902ec84431c1a8e5fe4b096470f6bb317023f0a47887f2f8c093a3e","source":"provider_usage","unbilledReason":null,"version":"provider-billing-v1"},"probeId":"probe-ko-business","stage":"rewrite"},{"attemptIndex":1,"billingEvidence":{"billed":true,"externalReferenceSha256":"f6008f56893a8fca9bcea1b9a4ba5d7cc66c47d8aeba66ca5c42a226d35281b8","provider":"claude","providerReportedAmountUsdMicros":null,"rawUsageSha256":"ace7849a45fd1080108886eaa958e0e50365bb5ca9d53a55366d6fd607a371f7","source":"provider_usage","unbilledReason":null,"version":"provider-billing-v1"},"probeId":"probe-ko-business","stage":"mps"},{"attemptIndex":1,"billingEvidence":{"billed":true,"externalReferenceSha256":"c964f946ad8c49941e4a650311b38bdf34839ecd0ca7387aed9e6f2467b0078b","provider":"claude","providerReportedAmountUsdMicros":null,"rawUsageSha256":"3ef92f5a996e73f3f9a90cb9d119e4c7e10fbe132b1f14d849810fc9f881b436","source":"provider_usage","unbilledReason":null,"version":"provider-billing-v1"},"probeId":"probe-ko-business","stage":"fidelity"},{"attemptIndex":1,"billingEvidence":{"billed":true,"externalReferenceSha256":"37b05cfd3ccec24ead713a5c9af83b312ae44b0f599c254fd1c2d2186ed3873c","provider":"claude","providerReportedAmountUsdMicros":null,"rawUsageSha256":"2442c8a642010a7b0ced81052aad2a00da3dd82fe60aedd0809f99a52371bbf2","source":"provider_usage","unbilledReason":null,"version":"provider-billing-v1"},"probeId":"probe-en-blog","stage":"rewrite"},{"attemptIndex":1,"billingEvidence":{"billed":true,"externalReferenceSha256":"bb94bbd1e1159cf1cda891c1114ecce03b73d0390520bc413d9069d6a0043f6c","provider":"claude","providerReportedAmountUsdMicros":null,"rawUsageSha256":"4fe2ea65cca538d356ce6035cdb0dd08a921125a24ca403ca8c90a473f0eccde","source":"provider_usage","unbilledReason":null,"version":"provider-billing-v1"},"probeId":"probe-en-blog","stage":"mps"},{"attemptIndex":1,"billingEvidence":{"billed":true,"externalReferenceSha256":"92e8fba7cb9d58410b973743502e3b02168c0d23cbf74a7825774df2223c1546","provider":"claude","providerReportedAmountUsdMicros":null,"rawUsageSha256":"10279aac6cf29c96d349a9feec79862e62ebf2290a267be5cadef86668bf4f05","source":"provider_usage","unbilledReason":null,"version":"provider-billing-v1"},"probeId":"probe-en-blog","stage":"fidelity"},{"attemptIndex":1,"billingEvidence":{"billed":true,"externalReferenceSha256":"fcf372f2bd89e974c17982ecc653e7a65fe715609828034765001e532c6a03c6","provider":"claude","providerReportedAmountUsdMicros":null,"rawUsageSha256":"15f545ed4c2c9dc15ab348b2a59fb659229898a4bfe6d49f73e3f6c20dd74e8f","source":"provider_usage","unbilledReason":null,"version":"provider-billing-v1"},"probeId":"probe-ko-sns","stage":"rewrite"},{"attemptIndex":1,"billingEvidence":{"billed":true,"externalReferenceSha256":"9a5f714c86f037ea0b2edfbacecf14c35eba1a4c2b26059f7e2bd6e577b24bef","provider":"claude","providerReportedAmountUsdMicros":null,"rawUsageSha256":"fb21832e293e5bf1df6f5f5084f7c8522a6ea8ffebe8a5a1eb982e8088c316f7","source":"provider_usage","unbilledReason":null,"version":"provider-billing-v1"},"probeId":"probe-ko-sns","stage":"mps"},{"attemptIndex":1,"billingEvidence":{"billed":true,"externalReferenceSha256":"4617290741df614352637ad60939b6f4126e83fb13c796e312c2608fd39c7128","provider":"claude","providerReportedAmountUsdMicros":null,"rawUsageSha256":"bcd07e479c27991a76867f20f024f61b44ce3ab7d52c6df221d340e269409b1d","source":"provider_usage","unbilledReason":null,"version":"provider-billing-v1"},"probeId":"probe-ko-sns","stage":"fidelity"}],"rawG002":{"channel":"staging","collectorVersion":"g002-inproc-v1","deploymentId":"local-2c084b1708a8ec9461b05b33fe196fbbe2604219","effectiveModel":"claude-sonnet-5","probes":[{"id":"probe-ko-business","inputChars":341,"stages":{"fidelity":[{"attemptIndex":1,"effectiveModel":"claude-sonnet-5","minimumChargeApplied":false,"outcome":"success","requestedModel":"claude-sonnet-5","retryReason":"initial","usage":{"cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"inference_geo":"global","input_tokens":933,"output_tokens":640,"output_tokens_details":{"thinking_tokens":455},"service_tier":"standard"}}],"mps":[{"attemptIndex":1,"effectiveModel":"claude-sonnet-5","minimumChargeApplied":false,"outcome":"success","requestedModel":"claude-sonnet-5","retryReason":"initial","usage":{"cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"inference_geo":"global","input_tokens":1610,"output_tokens":1559,"output_tokens_details":{"thinking_tokens":1239},"service_tier":"standard"}}],"rewrite":[{"attemptIndex":1,"effectiveModel":"claude-sonnet-5","minimumChargeApplied":false,"outcome":"success","requestedModel":"claude-sonnet-5","retryReason":"initial","usage":{"cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"cache_creation_input_tokens":0,"cache_read_input_tokens":34254,"inference_geo":"global","input_tokens":416,"output_tokens":4580,"output_tokens_details":{"thinking_tokens":3869},"service_tier":"standard"}}]}},{"id":"probe-en-blog","inputChars":732,"stages":{"fidelity":[{"attemptIndex":1,"effectiveModel":"claude-sonnet-5","minimumChargeApplied":false,"outcome":"success","requestedModel":"claude-sonnet-5","retryReason":"initial","usage":{"cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"inference_geo":"global","input_tokens":777,"output_tokens":224,"output_tokens_details":{"thinking_tokens":92},"service_tier":"standard"}}],"mps":[{"attemptIndex":1,"effectiveModel":"claude-sonnet-5","minimumChargeApplied":false,"outcome":"success","requestedModel":"claude-sonnet-5","retryReason":"initial","usage":{"cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"inference_geo":"global","input_tokens":1454,"output_tokens":1330,"output_tokens_details":{"thinking_tokens":916},"service_tier":"standard"}}],"rewrite":[{"attemptIndex":1,"effectiveModel":"claude-sonnet-5","minimumChargeApplied":false,"outcome":"success","requestedModel":"claude-sonnet-5","retryReason":"initial","usage":{"cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"cache_creation_input_tokens":0,"cache_read_input_tokens":26787,"inference_geo":"global","input_tokens":295,"output_tokens":2465,"output_tokens_details":{"thinking_tokens":1935},"service_tier":"standard"}}]}},{"id":"probe-ko-sns","inputChars":230,"stages":{"fidelity":[{"attemptIndex":1,"effectiveModel":"claude-sonnet-5","minimumChargeApplied":false,"outcome":"success","requestedModel":"claude-sonnet-5","retryReason":"initial","usage":{"cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"inference_geo":"global","input_tokens":769,"output_tokens":203,"output_tokens_details":{"thinking_tokens":0},"service_tier":"standard"}}],"mps":[{"attemptIndex":1,"effectiveModel":"claude-sonnet-5","minimumChargeApplied":false,"outcome":"success","requestedModel":"claude-sonnet-5","retryReason":"initial","usage":{"cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"inference_geo":"global","input_tokens":1446,"output_tokens":1562,"output_tokens_details":{"thinking_tokens":1124},"service_tier":"standard"}}],"rewrite":[{"attemptIndex":1,"effectiveModel":"claude-sonnet-5","minimumChargeApplied":false,"outcome":"success","requestedModel":"claude-sonnet-5","retryReason":"initial","usage":{"cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"cache_creation_input_tokens":0,"cache_read_input_tokens":34254,"inference_geo":"global","input_tokens":310,"output_tokens":1760,"output_tokens_details":{"thinking_tokens":1219},"service_tier":"standard"}}]}}],"provider":"claude","requestedModel":"claude-sonnet-5","sourceCommitSha":"2c084b1708a8ec9461b05b33fe196fbbe2604219"}} diff --git a/docs/operations/pay-b-cost-20260724.json.bundle.json b/docs/operations/pay-b-cost-20260724.json.bundle.json new file mode 100644 index 0000000..6ff52be --- /dev/null +++ b/docs/operations/pay-b-cost-20260724.json.bundle.json @@ -0,0 +1 @@ +{"financialInputs":{"bootstrap":{"confidenceBps":9500,"iterations":10000,"seed":"PAY-B-20260723-1236551-1932893"},"feeUsdMicros":999500,"refundReserveUsdMicros":499500,"unitChars":1000000},"pricing":{"cacheCreationBillingGranularityTokens":1,"cacheCreationUsdMicrosPerMillionTokens":3750000,"cacheReadBillingGranularityTokens":1,"cacheReadUsdMicrosPerMillionTokens":300000,"inputBillingGranularityTokens":1,"inputUsdMicrosPerMillionTokens":3000000,"minimumChargeUsdMicros":1,"outputBillingGranularityTokens":1,"outputUsdMicrosPerMillionTokens":15000000,"source":"https://platform.claude.com/docs/en/about-claude/pricing","sourceChars":"Claude Sonnet 5 starting September 1, 2026: Base Input $3 / MTok; 5m Cache Writes $3.75 / MTok; 1h Cache Writes $6 / MTok; Cache Hits & Refreshes $0.30 / MTok; Output $15 / MTok. Introductory pricing of $2/$10 per million input/output tokens is in effect through August 31, 2026, after which the standard pricing of $3/$15 per million input/output tokens will take effect. Retrieved 2026-07-24 from https://platform.claude.com/docs/en/about-claude/pricing (model pricing table). Standard post-introductory rates are used as the base for margin evaluation.","sourceSha256":"8bafd83ba9b47388be151750679a349be763fb74b45671c0740678ade4d80a59"},"providerBillingFacts":[{"attemptIndex":1,"billingEvidence":{"billed":false,"externalReferenceSha256":"45078a95036ffd7977a9cfd2a28976919333c637b0cc2f38e510a5c9f23a0176","provider":"claude","providerReportedAmountUsdMicros":null,"rawUsageSha256":null,"source":"provider_usage","unbilledReason":"provider_error_without_usage","version":"provider-billing-v1"},"probeId":"probe-ko-business","stage":"rewrite"},{"attemptIndex":2,"billingEvidence":{"billed":true,"externalReferenceSha256":"1dccde91ff6bdefcd58f5575238ad967510d18bcb0ec3f702f50299de3a16e5f","provider":"claude","providerReportedAmountUsdMicros":null,"rawUsageSha256":"95576537ced8b7b4b934fcb47f48f7a50056f05ac81afeb1c4d87796a2d53b2b","source":"provider_usage","unbilledReason":null,"version":"provider-billing-v1"},"probeId":"probe-ko-business","stage":"rewrite"},{"attemptIndex":1,"billingEvidence":{"billed":true,"externalReferenceSha256":"4cc177b8afa6baabb8c5a30a5f2d0e3474d03c8e85d211317ce6300bb0f77054","provider":"claude","providerReportedAmountUsdMicros":null,"rawUsageSha256":"04e09c28e924685923acd385b73a5d0d134bfbf137af6dc708dfa362ac2e9183","source":"provider_usage","unbilledReason":null,"version":"provider-billing-v1"},"probeId":"probe-ko-business","stage":"mps"},{"attemptIndex":1,"billingEvidence":{"billed":true,"externalReferenceSha256":"83b58f3456bb78ac6f9128840abcd7dcfcd519acce6baf88b29a0b215575d69b","provider":"claude","providerReportedAmountUsdMicros":null,"rawUsageSha256":"a5f7204aaae433fbf98a4de5e7c3b85fec63041e67612a41b634527cf7e279f7","source":"provider_usage","unbilledReason":null,"version":"provider-billing-v1"},"probeId":"probe-ko-business","stage":"fidelity"},{"attemptIndex":1,"billingEvidence":{"billed":true,"externalReferenceSha256":"c7aa96932a10de2e42596f25e4bb8b030a84cc6c2951a30478f7036ca06f2cde","provider":"claude","providerReportedAmountUsdMicros":null,"rawUsageSha256":"ba8d13aa4832d8cd10d618d2b4a9895a5b4e3328c58a4f916f7a62de6131393f","source":"provider_usage","unbilledReason":null,"version":"provider-billing-v1"},"probeId":"probe-en-blog","stage":"rewrite"},{"attemptIndex":1,"billingEvidence":{"billed":true,"externalReferenceSha256":"a0e16296d1377a1133c9740f4345932b5abefdd4d7d797857098b5c325a1b878","provider":"claude","providerReportedAmountUsdMicros":null,"rawUsageSha256":"ebc53065001b2f65abf37b283070026d60f9596d33e32eb503c3f2632f280db7","source":"provider_usage","unbilledReason":null,"version":"provider-billing-v1"},"probeId":"probe-en-blog","stage":"mps"},{"attemptIndex":1,"billingEvidence":{"billed":true,"externalReferenceSha256":"2e0684dc32bb5595621be6495ce4e25ef9fc1b2f9ee58349a50c682f39706bed","provider":"claude","providerReportedAmountUsdMicros":null,"rawUsageSha256":"0df150a0b3dbbc006a94b86083e751e915b353fbc1f9a6fcabc3ff6abe2a38e8","source":"provider_usage","unbilledReason":null,"version":"provider-billing-v1"},"probeId":"probe-en-blog","stage":"fidelity"},{"attemptIndex":1,"billingEvidence":{"billed":true,"externalReferenceSha256":"6a51f056942d04d7bb2780b947ccc45c507af149b67fea79c71d56e5f44007b7","provider":"claude","providerReportedAmountUsdMicros":null,"rawUsageSha256":"46ebd4d87903177164aa55c44c25bcd26628c835f61effd70233cd75a170313d","source":"provider_usage","unbilledReason":null,"version":"provider-billing-v1"},"probeId":"probe-ko-sns","stage":"rewrite"},{"attemptIndex":1,"billingEvidence":{"billed":true,"externalReferenceSha256":"54fae47a5d2dc1eb43c5d3d1334178025f5eb353c322df973898971fe9ff98be","provider":"claude","providerReportedAmountUsdMicros":null,"rawUsageSha256":"a038263febd51fb11b267600dbcc59de6f7eee6fc5414dc5af838f2d76981a45","source":"provider_usage","unbilledReason":null,"version":"provider-billing-v1"},"probeId":"probe-ko-sns","stage":"mps"},{"attemptIndex":1,"billingEvidence":{"billed":true,"externalReferenceSha256":"f7f0ebd5b1c1f508fdd842692411302eead800995a9864a1e5c1eff465c17fca","provider":"claude","providerReportedAmountUsdMicros":null,"rawUsageSha256":"54d9a9f2dcff26275b79ebcc6235528acc25b136bfe8b6039ced086a94c37d62","source":"provider_usage","unbilledReason":null,"version":"provider-billing-v1"},"probeId":"probe-ko-sns","stage":"fidelity"}],"rawG002":{"channel":"staging","collectorVersion":"g002-inproc-v1","deploymentId":"local-afde4e6e7334b2db90c76c8ed4ad0ed4f59bb18d","effectiveModel":"claude-sonnet-5","probes":[{"id":"probe-ko-business","inputChars":341,"stages":{"fidelity":[{"attemptIndex":1,"effectiveModel":"claude-sonnet-5","minimumChargeApplied":false,"outcome":"success","requestedModel":"claude-sonnet-5","retryReason":"initial","usage":{"completion_tokens":158,"prompt_tokens":906,"total_tokens":1064}}],"mps":[{"attemptIndex":1,"effectiveModel":"claude-sonnet-5","minimumChargeApplied":false,"outcome":"success","requestedModel":"claude-sonnet-5","retryReason":"initial","usage":{"completion_tokens":1633,"prompt_tokens":1583,"total_tokens":3216}}],"rewrite":[{"attemptIndex":1,"effectiveModel":null,"minimumChargeApplied":false,"outcome":"error","requestedModel":"claude-sonnet-5","retryReason":"initial","usage":null},{"attemptIndex":2,"effectiveModel":"claude-sonnet-5","minimumChargeApplied":false,"outcome":"success","requestedModel":"claude-sonnet-5","retryReason":"temperature_schema","usage":{"completion_tokens":3990,"prompt_tokens":34670,"total_tokens":38660}}]}},{"id":"probe-en-blog","inputChars":732,"stages":{"fidelity":[{"attemptIndex":1,"effectiveModel":"claude-sonnet-5","minimumChargeApplied":false,"outcome":"success","requestedModel":"claude-sonnet-5","retryReason":"initial","usage":{"completion_tokens":342,"prompt_tokens":717,"total_tokens":1059}}],"mps":[{"attemptIndex":1,"effectiveModel":"claude-sonnet-5","minimumChargeApplied":false,"outcome":"success","requestedModel":"claude-sonnet-5","retryReason":"initial","usage":{"completion_tokens":1224,"prompt_tokens":1394,"total_tokens":2618}}],"rewrite":[{"attemptIndex":1,"effectiveModel":"claude-sonnet-5","minimumChargeApplied":false,"outcome":"success","requestedModel":"claude-sonnet-5","retryReason":"initial","usage":{"completion_tokens":994,"prompt_tokens":27082,"total_tokens":28076}}]}},{"id":"probe-ko-sns","inputChars":230,"stages":{"fidelity":[{"attemptIndex":1,"effectiveModel":"claude-sonnet-5","minimumChargeApplied":false,"outcome":"success","requestedModel":"claude-sonnet-5","retryReason":"initial","usage":{"completion_tokens":137,"prompt_tokens":780,"total_tokens":917}}],"mps":[{"attemptIndex":1,"effectiveModel":"claude-sonnet-5","minimumChargeApplied":false,"outcome":"success","requestedModel":"claude-sonnet-5","retryReason":"initial","usage":{"completion_tokens":1698,"prompt_tokens":1457,"total_tokens":3155}}],"rewrite":[{"attemptIndex":1,"effectiveModel":"claude-sonnet-5","minimumChargeApplied":false,"outcome":"success","requestedModel":"claude-sonnet-5","retryReason":"initial","usage":{"completion_tokens":2027,"prompt_tokens":34564,"total_tokens":36591}}]}}],"provider":"claude","requestedModel":"claude-sonnet-5","sourceCommitSha":"afde4e6e7334b2db90c76c8ed4ad0ed4f59bb18d"}} diff --git a/docs/operations/register-failure-handoff-20260726.md b/docs/operations/register-failure-handoff-20260726.md new file mode 100644 index 0000000..22b39f6 --- /dev/null +++ b/docs/operations/register-failure-handoff-20260726.md @@ -0,0 +1,181 @@ +# Resolved: the register failures were a scoring bug (2026-07-26 → 07-27) + +**Status: closed on 2026-07-27.** The premise of this document — five registers +failing on every engine, unmovable by engine choice — was an artifact of the +measuring apparatus. Two defects in it, once fixed, took the same engine on the +same 22 fixtures from 9/22 to 20/22, and Korean from 2/11 to 11/11. What +remains is two English fixtures, described at the end. + +## Outcome after both fixes + +| | broken rubric | fidelity fixed | + prompt parity | +|---|---:|---:|---:| +| pass | 9/22 (41%) | 17/22 (77%) | **20/22 (91%)** | +| MPS mean | 76.3 | 80.5 | **90.4** | +| ko | 2/11 | 5/11 | **11/11** | + +Two defects, both in the measuring apparatus rather than the product: + +1. **The fidelity rubric** charged removal of marketing hype as omitted claims, + required the rewritten register to match the original, and penalized the + shortening filler removal causes. It was failing correct rewrites in + production, not only in the harness. +2. **The harness prompt** omitted the persona. v6.2 made the persona the sole + voice owner and both shipping surfaces moved with it; the harness did not, + so for Korean it dropped the directive ordering the model to preserve every + claim, figure, and quotation. Meaning was being measured on rewrites that + were never told to preserve meaning. + +Every register this document called systematically broken now passes: +`ko-instructional` MPS 20 → 100, `ko-marketing` 50 → 100, `ko-social` 40 → 100, +`ko-blog` 45 → 70, `ko-product` 50 → 80. Final run used the subscription seats +(`gemini-3.6-flash` rewriting, `gpt-5.5` judging) and cost nothing. + +## What is actually left + +Two fixtures, both English, and they fail in opposite directions: + +| fixture | status | MPS | fidelity | AI score | +|---|---|---:|---:|---| +| en-marketing-01 | fail | 60 | 75 | 35.6 → 5.7 | +| en-public-docs-01 | warn | 70 | 91.7 | 15.6 → 16.5 | + +`en-marketing-01` strips hype thoroughly — the AI score falls from 35.6 to 5.7 +— but loses anchors doing it. `en-public-docs-01` is the reverse: meaning +holds, and the AI score does not improve, so the rewrite is too timid there. + +Neither is a register-wide failure and both are single fixtures. Add a second +fixture per register before treating either as a pattern; this document is +itself the record of what happens when one fixture is read as a trend. + +## Historical record: the original observation + +Everything below is the 2026-07-26 analysis, kept because it shows how the +rubric bug presented and how it was traced. Its conclusions are superseded. + +## The observation + +Two unrelated frontier engines were run over all 22 live-quality fixtures with +the same fixed independent judge (`gpt-5.3-chat-latest`, calibrated AUC 0.99). +They fail the same 11 fixtures: + +| fixture | gemini-3.6-flash | claude-sonnet-5 | +|---|---|---| +| en-blog-01 | mps 85 / fid 58.3 | mps 20 / fid 25 | +| en-instructional-01 | mps 83.3 / fid 41.7 | mps 100 / fid 58.3 | +| en-marketing-01 | mps 80 / fid 50 | mps 60 / fid 33.3 | +| en-product-01 | mps 40 / fid 41.7 | mps 40 / fid 41.7 | +| en-public-docs-01 | mps 66.7 / fid 58.3 | mps 85.7 / fid 66.7 | +| en-social-01 | scoring error | mps 76 / fid 66.7 | +| ko-blog-01 | mps 50 / fid 50 | mps 64 / fid 66.7 | +| ko-instructional-01 | mps 20 / fid 58.3 | mps 64 / fid 50 | +| ko-marketing-01 | mps 75 / fid 41.7 | mps 48 / fid 50 | +| ko-product-01 | mps 66.7 / fid 66.7 | mps 80 / fid 50 | +| ko-social-01 | mps 40 / fid 75 | mps 42 / fid 58.3 | + +Both pass the same 6: `en-chat-01`, `en-howto-01`, `en-news-01`, `ko-chat-01`, +`ko-email-01`, `ko-public-docs-01`. + +Read the failing set by register and the shape is obvious: blog, instructional, +marketing, product, and social all fail on both engines, while everything that +passes is either conversational or documentary. It is register-shaped, not +model-shaped. + +## Why this is the bigger lever + +Every engine measured on 2026-07-25 and 07-26 lands between 36% and 45% pass on +these fixtures. That range holds across a twentyfold price spread, from +deepseek-v4-flash at $0.003 per rewrite up to claude-sonnet-5 at $0.156. When a +ceiling ignores both price and model family that completely, whatever sets it +is not the engine — it is the prompt, or the gate that scores the prompt's +output. + +**Fidelity is the blocker, not meaning loss.** Of the 11 shared failures, 9 +sit below the fidelity floor. In 7 of those, MPS is 66.7 or higher. One case +is plainly self-contradictory: sonnet-5 on `en-instructional-01` scored MPS +100 with fidelity 58.3. Meaning fully preserved, yet judged unfaithful. + +## What would overturn this + +One run, one judge, 22 fixtures, one fixture per register. That is thin enough +that a single mis-specified fixture can look like a register-wide failure, so +treat the register grouping as a lead rather than a finding. Two checks would +settle it. Add a second fixture for each failing register and see whether the +failures follow the register or stay stuck to the individual file. Then rerun +the same 22 under a different fixed judge — if the failures move, the problem +lives in the judge, and hypothesis 1 below is already most of the answer. + +Worth remembering how the earlier reads went wrong this week. A 6-fixture pass +suggested gemini-3.6-flash never dropped below MPS 80, and the 22-fixture rerun +put its floor at 20. Small samples in this harness have produced confident, +wrong conclusions twice already. + +## Hypothesis 1 is confirmed — it is a scoring bug, and it is live + +Hypothesis 1 below was checked on 2026-07-27 and holds. A real request to the +production free tier returned an error to the user, `floor_failed` on fidelity, +for a rewrite that was correct: + +- original: "빠르게 변화하는 디지털 환경 속에서, 본 솔루션은 **혁신적인 시너지**를 + 활용하여 고객에게 **전례 없는** 가치를 **원활하게** 제공합니다." +- rewrite: "디지털 환경이 빠르게 변하고 있다. 이 솔루션은 고객에게 새로운 가치를 + 제공한다." +- MPS: 100. Fidelity: failed, `length_ratio_pct` 62, rationale "omits the + specific claims about 'innovative synergy' and 'seamless delivery'". + +The rewrite removed exactly the marketing packaging patina exists to remove, +and the fidelity judge charged it as omitted claims. Reading the prompt in +`src/scoring.js` shows three separate mechanisms doing this: + +1. `claims_preserved` has no stylistic-packaging exemption. `scoreMPS` has one + and states it explicitly ("removing or toning them down is the rewrite's + job and must never be penalized"); `scoreFidelity` never received it. +2. `tone_match` scores whether "register/formality of REWRITTEN matches + ORIGINAL". Changing an AI-sounding register is the product's entire + function, so this criterion penalizes success by construction. +3. `length_ratio` penalizes shortening. Removing filler shortens text, so + every clean rewrite of hype-dense copy loses points here too. + +Consequences: hype-dense registers (marketing, social, product, blog) cannot +pass regardless of engine, which is exactly the observed failure set; the free +tier returns errors instead of rewrites on the copy most likely to be pasted +into a humanizer; and this week's engine comparisons are biased against +engines that strip hype most thoroughly. + +Fixing this changes what the product accepts, so it is an owner decision, not +a silent patch. The minimal change is to give `scoreFidelity` the same +packaging exemption `scoreMPS` already has, and to reconsider whether +`tone_match` and `length_ratio` belong in a gate for a tool whose job is to +change tone and cut filler. + +## Remaining hypotheses + +2. **The fixtures are mis-specified.** `*-product-01` fails on both engines + with MPS 40 — check whether those fixtures carry dense factual anchors + (spec lists, numbers) that no humanizing rewrite can retain while changing + voice, i.e. the fixture asks for something contradictory. +3. **The prompt lacks register handling.** Profiles exist for some registers; + confirm which profile these fixtures request and whether the pattern packs + have anything for instructional/product prose. + +## Next steps + +1. Decide on the `scoreFidelity` rubric (owner call — it changes what the + product accepts). Minimal version: port the packaging exemption from + `scoreMPS`, and drop or rescope `tone_match`. +2. Rerun the 22-fixture comparison after any rubric change. The current + engine numbers were produced under the biased gate and should not be cited + afterwards. +3. Only then decide whether the fidelity floor of 70 is calibrated. + +## What is already settled (do not redo) + +- Judge selection: `docs/research/2026-judge-calibration.md`. +- Engine cost/quality comparison and prompt-cache economics: + `docs/operations/serving-engine-cost-20260725.md`. +- `gemini-3.6-flash` is allowlisted and is the documented default for both + tiers as of 2026-07-26; the runtime switch is a hosting env change + (`PATINA_PRO_PROVIDER/MODEL`, `PATINA_FREE_PROVIDER/MODEL`). +- The free tier previously served `gpt-4.1-mini`, which measured an AI-score + improvement of 0.2 — effectively no rewrite. That is why the free default + moved too. diff --git a/docs/operations/serving-engine-cost-20260725.md b/docs/operations/serving-engine-cost-20260725.md new file mode 100644 index 0000000..c0e8cd4 --- /dev/null +++ b/docs/operations/serving-engine-cost-20260725.md @@ -0,0 +1,194 @@ +# Serving-engine cost and quality measurements (2026-07-25) + +Measured to answer one question: what should serve the Pro tier and the free +trial, given that the incumbent (`claude-sonnet-5`) consumes essentially the +whole $9.99 subscription when one license exhausts its monthly character cap. + +Not a Gate-B artifact, not an approval, and not a decision to change any +provider default. Provider defaults and launch bindings stay frozen where the +v6.4 preflight hold pins them. + +## Why the prompt is unusually cache-friendly + +The rewrite prompt is a fixed prefix (pattern catalog + profile + voice) with +only the user text after the data fence: + +| language | total prompt | cacheable prefix | user text | +|---|---:|---:|---:| +| ko | 41,651 chars | 41,517 (99.7%) | 134 | +| en | 72,561 chars | 72,427 (99.8%) | 134 | +| zh | 35,294 chars | 35,160 (99.6%) | 134 | +| ja | 38,639 chars | 38,505 (99.7%) | 134 | + +Cost is therefore dominated by input tokens that are identical on every +request, which makes prompt caching — not model choice — the first lever. + +## Prompt caching, measured on the real path + +`PATINA_ANTHROPIC_NATIVE_CACHE=1`, ko web prompt, `claude-sonnet-5`, two +identical calls: + +| call | input | cache write | cache read | latency | +|---|---:|---:|---:|---:| +| 1st | 147 | 34,254 | 0 | 25.2s | +| 2nd | 147 | 0 | **34,254** | **12.0s** | + +Cost per request at the checked-in sonnet-5 rates ($3 in / $15 out / +$0.30 cache read / $3.75 5m cache write), 1,500 output tokens assumed: + +| state | cost | vs uncached | +|---|---:|---:| +| uncached | $0.1257 | — | +| cache miss (write) | $0.1514 | +20% | +| cache hit (read) | $0.0332 | **−74%** | + +Break-even hit rate is **21.7%**. With the 5-minute ephemeral TTL refreshing +on each hit, that is roughly **3 requests/hour per language variant**; the four +languages hold separate cache entries, so low-traffic variants stay in the +surcharge region. Latency halves on every hit regardless of traffic. + +| hit rate | cost/request | monthly cap exhausted (50 requests) | +|---|---:|---:| +| 0% | $0.1514 | $7.57 | +| 50% | $0.0923 | $4.62 | +| 90% | $0.0450 | $2.25 | + +## Candidate engines, same fixtures and same judge + +6 fixtures (ko 3 + en 3), fixed judge `grok-4.5` (calibrated AUC 0.93, +independent of every candidate). Cost uses measured tokens (~18.8k in / ~280 +out) at published rates; `gpt-5.3-chat-latest` has no published rate (the 5.3 +line is absent from the pricing table) and is interpolated from 5.2/5.4. + +| engine | pass | MPS mean | **MPS worst** | fidelity mean | s/rewrite | $/rewrite | +|---|---|---:|---:|---:|---:|---:| +| gemini-3.6-flash | 3/6 | **91.9** | **80.0** | **68.0** | 8.3 | $0.0302 | +| deepseek-v4-flash | 3/6 | 77.2 | 15.0 | 61.1 | 14.1 | **$0.0032** | +| gpt-5.3-chat-latest | 2/6 | 70.5 | 15.0 | 61.1 | **4.2** | $0.0404 | + +The pass counts are not an absolute quality verdict — this gate is strict +enough that `claude-sonnet-4-6` scored 2/3 and `claude-sonnet-5` 1/3 on the ko +subset, and the blog register fails for nearly every model. Only the relative +comparison is meaningful. + +**Worst-case MPS is the deciding column.** deepseek-v4-flash is 10x cheaper +than gemini-3.6-flash and ties on pass count, but it dropped to MPS 15 on +`ko-blog-01` (most of the original meaning gone). gpt-5.3-chat-latest did the +same. + +## Expanded run (2026-07-26): the 6-fixture read was wrong + +Rerun on all 22 fixtures with `gpt-5.3-chat-latest` as the fixed judge +(calibrated AUC 0.99, independent of both engines). Two 6-fixture claims from +the section above did not survive: + +| engine | pass | AI-score delta | MPS mean | MPS worst | fidelity mean | meaning-loss fixtures | $/rewrite | +|---|---|---:|---:|---:|---:|---:|---:| +| gemini-3.6-flash | 9/22 | **13.0** | 76.3 | 20 | 69.4 | **7** | **$0.030** | +| claude-sonnet-5 (Pro pin) | 8/22 | 11.8 | 73.1 | 20 | 65.2 | 10 | $0.156 | +| gpt-4.1-mini (free default) | 10/22 | **0.2** | 84.8 | 33.3 | 81.8 | 4 | $0.008 | + +1. "gemini-3.6-flash never falls below MPS 80" was small-sample luck. On 22 + fixtures its worst case is 20 (`ko-instructional-01`), the same floor as + sonnet-5. It still loses meaning on fewer fixtures (7 vs 10) and improves + the AI score more (13.0 vs 11.8) at a fifth of the cost and a third of the + latency, so the ranking holds — but on margin, not dominance. +2. `gpt-4.1-mini` topping the pass/MPS columns is an artifact, not a win. Its + AI-score delta is **0.2**: it returns the input essentially unchanged, so it + trivially preserves meaning while doing none of the product's work. Six of + its 22 results carry `ai_not_improved`. **The free tier is currently serving + no real rewrite.** Pass counts must always be read next to the AI delta. + +Both frontier engines fail the same registers (blog, instructional, social, +product, marketing) while passing email, public-docs, chat, and academic. Two +independent frontier models failing identically points at the prompt or the +gate for those registers, not at the models — that is the larger lever behind +the ~40% pass rate, and it is untouched by any engine swap. + +## Voided and re-run (2026-07-27) + +Every number above is void. Two defects sat under it: a fidelity rubric that +charged removal of stylistic packaging as omitted claims, and a harness prompt +built without the persona, so rewrites were graded on meaning they were never +instructed to preserve. Both penalized whichever engine stripped hype hardest — +the behaviour the product exists for. + +Rerun on the same 22 fixtures with both defects fixed, on subscription seats +(judge `gpt-5.5` via codex, candidates via their own seats, zero API spend): + +| | gemini-3.6-flash | claude-sonnet-5 | +|---|---:|---:| +| pass | **20/22** (ko 11/11, en 9/11) | **20/22** (ko 11/11, en 9/11) | +| MPS mean | 90.4 | 91.5 | +| fidelity mean | 92.4 | 94.7 | +| $/rewrite | **$0.030** | $0.156 | +| s/rewrite | **8.3** | 27.7 | + +**The two engines are level on quality.** Identical pass counts, and sonnet-5's +1.1-point MPS edge is inside one fixture of noise at n=22. The shipped choice +(gemini-3.6-flash on both tiers) holds, but the correct justification is not +"gemini rewrites better" — it is "quality is level, cost is 5x lower and +latency 3x lower". + +`en-marketing-01` fails on both engines, which points at that fixture or the +prompt rather than at either model. + +Cheaper candidates were rerun on the same fixed apparatus to see whether the +shipped engine could be undercut. It cannot, and the two failure modes are +mirror images: + +| engine | pass | MPS mean | MPS worst | meaning-loss | not-improved | $/rewrite | +|---|---|---:|---:|---:|---:|---:| +| gemini-3.6-flash | **20/22** | 90.4 | 60 | 1 | 1 | $0.030 | +| claude-sonnet-5 | **20/22** | 91.5 | 50 | 2 | 0 | $0.156 | +| deepseek-v4-flash | 18/22 | 83.9 | **24** | 4 | 1 | $0.003 | +| gemini-3.5-flash-lite | 10/22 | 84.9 | 40 | 4 | **8** | $0.007 | + +`deepseek-v4-flash` is ten times cheaper and still rewrites hard, but it gutted +`ko-news-01` to MPS 24 — roughly three quarters of a news item's content gone. +`gemini-3.5-flash-lite` fails the other way: eight fixtures carry +`ai_not_improved`, and four Korean ones score MPS 100 while the AI score +barely moves, meaning it returns the input nearly unchanged. That is the same +evasion `gpt-4.1-mini` showed and the reason its earlier apparent lead was an +artifact. + +Holding both ends at once is the actual difficulty of this task, and +`gemini-3.6-flash` is the cheapest model measured that does it. Treat $0.030 +per rewrite as the current floor rather than a number to shave. + +Still unmeasured on the fixed apparatus: `gpt-4.1-mini`, `gpt-5.4-mini`, +`gpt-5.3-chat-latest`, `gpt-5.6-luna`, `grok-4.3`. The OpenAI-hosted ones are +blocked on an exhausted account balance, not on method. None of them is in +production, and given that both cheaper models failed in opposite directions, +none is a promising cost lever. + +## Published rates gathered while comparing (per 1M tokens) + +| model | input | cached input | output | training on input | +|---|---:|---:|---:|---| +| deepseek-v4-flash | $0.14 | $0.0028 | $0.28 | — | +| deepseek-v4-pro | $0.435 | $0.0036 | $0.87 | — | +| grok-4.3 | $1.25 | $0.20 | $2.50 | — | +| grok-4.5 | $2.00 | $0.30 | $6.00 | — | +| kimi-k2.6 | $0.95 | $0.16 | $4.00 | — | +| kimi-k3 | $3.00 | $0.30 | $15.00 | — | +| gemini-3.6-flash (paid) | $1.50 | $0.15 | $7.50 | no | +| gemini-3.6-flash (batch) | $0.75 | $0.075 | $3.75 | no | +| gemini-3.6-flash (free tier) | $0 | — | $0 | **yes** | +| claude-sonnet-5 | $3.00 | $0.30 | $15.00 | no | + +The free Gemini tier trains on submitted content, so it cannot serve customer +text; the paid tier does not. Untested engines with a price advantage over +gemini-3.6-flash: `grok-4.3` ($0.024/rewrite) and `kimi-k2.6` ($0.019). GLM +and a per-token Qwen endpoint have no key in this environment. + +## Order of operations this implies + +1. Caching first. It is already implemented on both the buffered and streaming + paths, costs nothing to enable, carries no quality risk, and is worth up to + −74% once traffic clears ~3 requests/hour per language. +2. Engine comparison second, and on worst-case meaning preservation rather than + headline pass counts. +3. Any Pro provider/model change goes through the frozen-default process; the + contract allowlist (`PROVIDER_PRESETS`) currently offers neither + `gemini-3.6-flash` nor `gpt-5.3-chat-latest`. diff --git a/docs/operations/v6.4-preflight-hold.json b/docs/operations/v6.4-preflight-hold.json index cd6386c..c9e9a4c 100644 --- a/docs/operations/v6.4-preflight-hold.json +++ b/docs/operations/v6.4-preflight-hold.json @@ -33,10 +33,10 @@ "together": { "name": "together", "baseURL": "https://api.together.xyz/v1", "apiKeyEnv": "TOGETHER_API_KEY", "defaultModel": "meta-llama/Llama-3.3-70B-Instruct-Turbo-Free", "freeTier": true, "note": "Free models available (suffix \"-Free\"). Get a key at https://api.together.xyz/settings/api-keys" } }, "sourceHashes": { - ".env.example": "7f33aea2fbd9649c350d071c900c097c950d17ebf02f1fa758fab91ec9d504f6", + ".env.example": "3af551e2a9bd03674b766ece01a3b65ad68640acac4f66d23fc5020c0265ec30", "src/model-defaults.js": "c568977fcac8ea44d5387a8a8745b062675ec94d73d39ce528c492ad35f87176", "src/providers.js": "92415eacaf87da2d0f2aed7db97feeb98b8b087adfe584a02c4478353b807d90", - "src/web-rewrite-contract.js": "af4b970eb56c5738ffca6a00a7c03eb4366f1d67c3c9e6e7efc5a70a646484bc", + "src/web-rewrite-contract.js": "a0b187ed72cea6a16a6044aa339c5dd7b2b2c01caa0e0c33f4e7c04726ca5052", "playground/launch-config.js": "4d19fc8ce36651f73f94d80bbcd108a2ccbb50fa3fc25b54d75f193b42ac6bf4", "vercel.json": "37a54c0850db54e80eec963c570d4b8a047fcfb4bf56d1a133e5a3934b28260e", "scripts/checkout-evidence-bindings.mjs": "8dcdf4b23787f91c773b6d8b30857ae3bc90b9f6e1552561749b00c8502e1408", @@ -49,25 +49,25 @@ "docs/operations/pay-stg-runtime-20260716.json": "b0229e892b06e1ec303a001c1317c4c63fc3c98fd7e10243e64db07df4803d29", "docs/operations/pay-b-binding-20260723.json": "96eb8e0aba9fcb4ce67dd356bd35aaf678d8abea96a7ec134218eab7cd20f132", "tests/e2e/providers.test.js": "47958be678e9bbfd06bcdd849ec0df37ed765f886e245b7b528e7d596b6d9dfe", - "tests/unit/backend-model-defaults.test.js": "a9a0428bd53cb470505df58b25d7bd75d59200a7565809eb9f457fda1d22db29", + "tests/unit/backend-model-defaults.test.js": "908dd9b06a9d5305ad28b50007d6428435b0b6b7019d6a2210aff813bfb9cf3d", "tests/unit/web-deploy-invariants.test.js": "0c858c964a350115af6b567d8f3f707cc6749ede5e2967ab56f30b2fa0fb8bcd", "tests/unit/web-rewrite-contract.test.js": "d802df1b7f44ef05bcb11e3e68307f577da5529a406e5e92fa206d08bdcf2b2a", "tests/unit/web-rewrite-contract.redteam.test.js": "3c7591926e168b14f486199297fd1e84ed447c1b8d339c9b4860f79da0f6a7bb", "tests/unit/v6.4-preflight-hold.test.js": "a14660535c998cde52e8810fec7812305d1688af67927cc868524040c2eafd17", "tests/unit/v6.4-release-ready.test.js": "8d88aab35ce75bc40b4782932329c0402001afabc380ac349ccc1b82d01c12d7", - "package.json": "f630732601b46fbc7a4fdd49924a7444cc5ff94fa8b4efb3559df19562ee416b", - "package-lock.json": "d4a0814cc96fd2b46d9c5453345e41623a53149b7e5bdf5e79213101d700680c", + "package.json": "35eb4726142f428c90b4fb2fbfca614a2f2e59ca992a5a6a11e14f38b04eed68", + "package-lock.json": "ec897a402073b3fbf4972a1b8004a1db3ab8dbe17f507d9ecf2427755552d559", ".github/workflows/release.yml": "43900a0966a52c500d54b1971d4141fe2d380a893364b4cb7b9976e9e3795f8f", - "README.md": "f8b8cc52f646b44fc94da2f7ee3872303ae6cb52b81868423c9f9235987e91b8", - "README_KR.md": "eb8310a10040e5c3653344349d6feda5978f6b86db3e363dbe6af8f6575b3be1", - "README_ZH.md": "2fbeb8ef8dfbd4bf61b11ece14d551efc3d8b0274b39b28f1377046d92f46bdc", - "README_JA.md": "e7a59cc4a462f4a3b992bcacd49f801522f76270111a5080ee74a634db2ab2f0", - "SKILL.md": "5c8c09492bf01f88bc2eb088fb0e246b4fefb0b272b7af24dd97b202f8889b93", - ".patina.default.yaml": "540a12d5f6a2234fd348e2a22548bbb37bb293fbb03cecdfee16e745cae3eac8", - "packages/patina-humanizer/package.json": "0a21036548a4b2be6e2fb0d9412fa481fc2310b9e4ab2b8f0fd18505c897662c", - ".claude-plugin/plugin.json": "baa2ddf798791d7f4edacedd70bd6b32b15e68c505ab77dfd1bf20f6e73cac5e", - ".claude-plugin/marketplace.json": "95a2847f1248752eceb77762cf8d67e58d8405e291036372363f678040041e81", - "CHANGELOG.md": "cfc3af68e2723dd27d87c26f9c66aff4e0242bcbfd2772c39448dd6aa0ba45ef" + "README.md": "b02bdca10e47ab26ecc2c744a95f94e2e08c4975c3e068fd1958e1a4364f21ef", + "README_KR.md": "e6012d8dda75d26228ca4cb0b445d6273f5ea833bd4508fa9881c13f86279645", + "README_ZH.md": "9abc492e2a5387699e2bc9a049531774cc744a6bc94361711f013a9128824645", + "README_JA.md": "a03879584ab428fcde63e547057a21adb3ef912ed125e747f75ac95fd2ac0b4d", + "SKILL.md": "32ee49b87ef659fe90f84e443a0d847bec20b6e84176560762702a298b8c1a34", + ".patina.default.yaml": "96668f0f6194db2bd279efb6a51cf46162a4fca62f88cc7a417c4df80581ffa4", + "packages/patina-humanizer/package.json": "e47280ab9806c6d1bef271c9036010067f7964b8c312cbe0e8efa2cc5db2dfed", + ".claude-plugin/plugin.json": "fe9e6a0bcb1a310ca76c27deaa62910dafb2c4809afd98d062e9de2cc4415d37", + ".claude-plugin/marketplace.json": "6b629add04d793460fa2bee18f64584ab421eb0c9d85d594817fbdddf70a09ce", + "CHANGELOG.md": "468de6ae79bf14bc58d29e0c60ade46e6f6e87156e75e89c13143c4005384736" } }, "blockers": [ diff --git a/docs/research/2026-judge-calibration.md b/docs/research/2026-judge-calibration.md index c4fc47e..248b93f 100644 --- a/docs/research/2026-judge-calibration.md +++ b/docs/research/2026-judge-calibration.md @@ -40,6 +40,85 @@ family's generations by ~9 points (classic self-preference); grok is ~12 points *harsher* on its own family. The study series' cross-family judging rule already neutralizes this. +## Challenger round (2026-07-25) — cheap/fast HTTP judges + +Same corpus, same prompt, same pre-registered criteria. Motivation: the +live-quality judge probes found HTTP judges 4–9x faster and up to 100x +cheaper than the incumbent seats, with reasoning disabled where the provider +exposes a switch (reasoning was 93–95% of output tokens and bought nothing on +this structured verdict). Discrimination was unmeasured, so it was measured. +192/192 challenger calls parsed (0 lost); latency and token usage are recorded +per call in the same artifact. + +| judge | accuracy | AUC [95% CI] | bias human/AI | repeat SD | median s/call | verdict | +|---|---:|---|---|---:|---:|---| +| judge-gemini36flash (gemini-3.6-flash) | 0.91 | **0.96 [0.91, 1.00]** | 13.5 / 79.0 | 2.2 | 4.6 | **PASS** | +| judge-gpt53chat (gpt-5.3-chat-latest, non-reasoning) | 0.94 | **0.99 [0.95, 1.00]** | 39.5 / 86.1 | 2.9 | **2.3** | **PASS** | +| judge-grok420nr (grok-4.20-non-reasoning) | 0.56 | 0.71 [0.53, 0.86] | 19.7 / 23.6 | 2.7 | **0.7** | WATCH | +| judge-deepseek-nothink (deepseek-v4-flash, thinking off) | 0.50 | 0.70 [0.54, 0.84] | 21.3 / 30.4 | 12.5 | **0.2** | WATCH | + +**Speed bought nothing for the non-reasoning judges.** grok-4.20-nr called +**23 of 24** AI documents "human"; deepseek-nothink 20 of 24. Their human +false-positive rate is near zero precisely because they score almost +everything low: the human/AI mean gap is 4–9 points, versus 53–65 for every +PASS judge. Low repeat SD (2.7) is not stability here — it is the consistency +of a judge that always answers ~20. + +**Reasoning is not the requirement — tier is.** Both failures were +non-reasoning *and* lower tier, so a strong-tier non-reasoning chat model was +added as a confound separator: `gpt-5.3-chat-latest` reached accuracy 0.94, +AUC 0.99 [0.95, 1.00], repeat SD 2.9 at 2.3s per call — PASS, and the fastest +passing judge measured. A judge does not need a reasoning trace to read +Korean prose style; it needs to be a good model. Its self-preference is −24.4 +(markedly harsher on its own family's generations) — the conservative +direction for a gate, but it keeps the cross-family judging rule mandatory. + +Caveat that limits this specific pick: `gpt-5.3-chat-latest` is a **moving +alias** — the 5.3 line publishes no dated snapshot (unlike +`gpt-5.5-2026-04-23`), so this AUC is "the 5.3 chat model as served on +2026-07-25". If OpenAI repoints the alias, a "fixed" judge silently changes +underneath the comparison it exists to stabilize. Re-run this calibration +before trusting cross-run numbers from it, or use a judge that can be pinned +(gpt-5.5, gemini-3.6-flash) when the comparison must hold over time. Its unit +price is also unpublished (the 5.3 line is absent from the pricing table); +measured usage is 1,907 input / 64 output tokens per call. + +**gemini-3.6-flash is a real judge.** AUC 0.96 sits above grok-4.5 (0.93) and +below gpt-5.5 (1.00), with gpt-level repeat tightness (SD 2.2) and the +cleanest separation of the round (13.5 / 79.0). All 24 AI documents are +cross-family for it (no generator overlap), so no self-preference correction +applies. At 4.6s per call on a near-free tier it is the cost-effective judge +this round was looking for — as a **second seat**, not a replacement: gpt-5.5 +still holds the only 1.00. + +Operational note: the free Gemini tier may train on submitted text, so a +free-tier key is fine for this repo-owned fixture corpus and must not be used +to score customer text — that lane needs a paid no-training API judge +(`gpt-5.3-chat-latest` is the measured pick) or a subscription seat. + +## What this study does not measure (2026-07-27) + +Every number here is AI-likeness discrimination: can the judge tell AI prose +from human prose. The product gates on something else — meaning preservation +and fidelity — and that was never measured. The cost of the omission surfaced +on 2026-07-27, when a fidelity rubric that charged removal of marketing hype +as omitted claims was found returning `floor_failed` to production users for +correct rewrites, unnoticed for as long as it had shipped. + +`scripts/research/judge-rubric-check.mjs` closes the gap without human +labelling: ten constructed pairs whose correct verdict is fixed by +construction. Hype-only removal must pass; a changed figure, flipped negation, +dropped causal link, fabricated claim, or deleted claims must fail. First +results: + +| judge | rubric check | note | +|---|---|---| +| gpt-5.5 (codex seat) | **10/10** | caught the 120ms → 12ms swap at MPS 66.7 | +| gemini-3.6-flash | 9/10 | **accepted** the 120ms → 12ms swap at MPS 80 | + +A high AUC does not imply a judge grades meaning correctly. Run both checks +before adopting one. + ## Deterministic stylometry on the same corpus | layer | AUC [95% CI] | mean score human/AI | diff --git a/package-lock.json b/package-lock.json index a102ca4..cee6b70 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "patina-cli", - "version": "6.3.1", + "version": "6.3.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "patina-cli", - "version": "6.3.1", + "version": "6.3.2", "license": "MIT", "dependencies": { "js-yaml": "^4.1.0" diff --git a/package.json b/package.json index 51b4e66..5b3eecf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "patina-cli", - "version": "6.3.1", + "version": "6.3.2", "description": "AI text humanizer CLI — detects and removes AI writing patterns", "type": "module", "main": "src/cli.js", diff --git a/packages/patina-humanizer/package.json b/packages/patina-humanizer/package.json index 70d583b..7a6e251 100644 --- a/packages/patina-humanizer/package.json +++ b/packages/patina-humanizer/package.json @@ -1,13 +1,13 @@ { "name": "patina-humanizer", - "version": "6.3.1", + "version": "6.3.2", "description": "SEO-friendly alias for patina-cli", "type": "module", "bin": { "patina-humanizer": "bin/patina-humanizer.js" }, "dependencies": { - "patina-cli": "6.3.1" + "patina-cli": "6.3.2" }, "license": "MIT", "repository": { diff --git a/playground/chatgpt.css b/playground/chatgpt.css index 1d8fb0d..83f3191 100644 --- a/playground/chatgpt.css +++ b/playground/chatgpt.css @@ -348,7 +348,11 @@ a { color: inherit; } .msg__text { white-space: pre-wrap; word-break: break-word; } .msg__text--flagged { opacity: .72; } .msg__text--unapproved { border-inline-start: 2px solid var(--warn, #b7791f); padding-inline-start: 8px; } -.output-status { margin: 8px 0 0; color: var(--text-muted, var(--text-faint)); font-size: 13px; } +/* Screen-reader-only approval live region (role="status"). Sighted users already + get the flagged border + floor/error notes; the visible "unapproved/approved" + line read as an alarming warning during normal streaming, so it is clipped + out visually while staying announced to assistive tech. */ +.output-status { position: absolute; width: 1px; height: 1px; margin: -1px; padding: 0; overflow: hidden; clip: rect(0 0 0 0); clip-path: inset(50%); white-space: nowrap; border: 0; } .msg__text.streaming::after { content: "▍"; color: var(--accent); margin-left: 1px; animation: blink 1s step-start infinite; } @keyframes blink { 50% { opacity: 0; } } diff --git a/playground/chatgpt.js b/playground/chatgpt.js index 854f993..81da6d0 100644 --- a/playground/chatgpt.js +++ b/playground/chatgpt.js @@ -87,6 +87,7 @@ const I18N = { floorWarn: 'This rewrite didn’t pass patina’s meaning-preservation floor (MPS / fidelity), so it’s flagged. Try again or pick a stronger model.', reportFp: 'Flagged your own writing? Report a false positive →', failNote: 'Rewrite failed. Try again, or check the mode/key.', + numberSafetyNote: 'The rewrite didn’t keep the numbers/times exactly as written, so patina discarded it for safety. Try again.', quotaDaily: 'You’ve used today’s free quota. Try again tomorrow, or switch to BYOK mode with your own API key for unlimited use.', quotaHourly: 'Free quota is full for now. Try again shortly, or use BYOK mode with your own API key.', proUpsell: 'Upgrade to Pro — $9.99/mo', @@ -129,6 +130,7 @@ const I18N = { floorWarn: '이 리라이트는 patina의 의미 보존 기준(MPS·fidelity)을 통과하지 못해 경고로 표시했어요. 다시 시도하거나 더 강한 모델을 골라보세요.', reportFp: '직접 쓴 글인데 잡혔나요? 오탐 신고 →', failNote: '리라이트 실패. 다시 시도하거나 모드·키를 확인해 주세요.', + numberSafetyNote: '리라이트가 숫자·시간 표기를 원문 그대로 보존하지 못해 안전을 위해 결과를 폐기했어요. 다시 시도해 주세요.', quotaDaily: '오늘 무료 사용량을 다 쓰셨어요. 내일 다시 시도하거나, 본인 API 키로 BYOK 모드를 쓰면 제한 없이 이용할 수 있어요.', quotaHourly: '무료 사용량이 잠시 가득 찼어요. 잠시 후 다시 시도하거나, 본인 API 키로 BYOK 모드를 쓰면 바로 이용할 수 있어요.', proUpsell: 'Pro로 업그레이드 — $9.99/월', @@ -171,6 +173,7 @@ const I18N = { floorWarn: '该改写未通过 patina 的语义保留阈值（MPS·fidelity），已标记。请重试或选择更强的模型。', reportFp: '人工撰写却被标记？反馈误报 →', failNote: '改写失败。请重试，或检查模式 / 密钥。', + numberSafetyNote: '改写未能原样保留数字 / 时间，为安全起见已丢弃结果。请重试。', quotaDaily: '今天的免费额度已用完。请明天再试，或切换到 BYOK 模式使用自己的 API 密钥，即可无限制使用。', quotaHourly: '免费额度暂时已满。请稍后再试，或使用 BYOK 模式和自己的 API 密钥。', proUpsell: '升级到 Pro — 每月 $9.99', @@ -213,6 +216,7 @@ const I18N = { floorWarn: 'この書き換えは patina の意味保持しきい値（MPS・fidelity）を満たさず、警告表示しています。再試行するか、より強力なモデルを選んでください。', reportFp: '自分で書いた文章なのに検出？誤検出を報告 →', failNote: '書き換えに失敗しました。再試行するか、モード・キーを確認してください。', + numberSafetyNote: '書き換えが数値・時刻を原文どおりに保持できなかったため、安全のため結果を破棄しました。もう一度お試しください。', quotaDaily: '本日の無料利用枠を使い切りました。明日また試すか、ご自身のAPIキーでBYOKモードに切り替えると無制限で使えます。', quotaHourly: '無料利用枠が一時的にいっぱいです。しばらくして再試行するか、ご自身のAPIキーでBYOKモードをお使いください。', proUpsell: 'Proにアップグレード — 月額$9.99', @@ -1113,6 +1117,7 @@ function failureMessage(kind, ff, t) { case K.QUOTA_DAILY: return t.quotaDaily; case K.QUOTA_HOURLY: return t.quotaHourly; case K.QUOTA_CONCURRENT: return t.quotaConcurrent; + case K.NUMBER_SAFETY: return t.numberSafetyNote; case K.IP_UNAVAILABLE: case K.QUOTA_STORAGE: case K.QUOTA_SECRET: diff --git a/playground/index.html b/playground/index.html index fe1bc51..05599fe 100644 --- a/playground/index.html +++ b/playground/index.html @@ -219,7 +219,7 @@

Paste your own and see