fix: telemetry install identity churn

docs/posthog-identity-repair.md

@@ -0,0 +1,59 @@
+# PostHog Identity Repair
+
+AgentRail telemetry uses an anonymous `inst_<24 hex chars>` install ID as the PostHog `distinct_id`.
+Older CLI builds regenerated that ID during repeated setup flows, so one real install could appear as many
+PostHog people.
+
+## Forward Fix
+
+Release a CLI build that:
+
+- persists a home-scoped `telemetry-identity.json` file under the AgentRail home directory;
+- imports a valid existing `config.json.telemetry.installId` into that identity file;
+- reuses that identity for later `agentrail init` runs;
+- reports the package version from the current package metadata.
+
+Users must update before their future telemetry becomes stable:
+
+```sh
+npm install -g @agentrail-core/cli@latest
+```
+
+## Historical Repair
+
+PostHog can merge the known duplicate distinct IDs for a real install into one person by sending a
+`$merge_dangerously` capture event for each duplicate ID pair. Use this only after grouping IDs with
+strong evidence, such as the same time window, platform, AgentRail version, and geo/IP-derived cluster.
+
+Example payload shape:
+
+```json
+{
+  "event": "$merge_dangerously",
+  "distinct_id": "inst_canonical0000000000000000",
+  "properties": {
+    "$distinct_ids": [
+      "inst_canonical0000000000000000",
+      "inst_duplicate0000000000000000"
+    ]
+  }
+}
+```
+
+After merging, verify against person resolution, not a raw event export label. Historical event rows can
+still display their original `distinct_id`/person-looking label even after the person now owns many
+distinct IDs.
+
+Recommended verification:
+
+- In PostHog Persons, check the canonical person has all expected distinct IDs.
+- In HogQL, group by geo cluster and count distinct current person IDs for recent AgentRail events.
+- Treat exported `Person.display_name` values from old raw rows as historical labels, not proof that the
+merge failed.
+
+## Operational Notes
+
+- Keep the old-to-canonical mapping in a private incident note so the merge can be audited.
+- Do not merge across weak clusters. A bad merge is harder to recover from than leaving duplicate people.
+- Once users update, do not keep repairing new `inst_*` churn manually. New churn means the CLI identity
+fix is not working or users are still on an older build.

src/cli/agentrail-home.ts

@@ -6,6 +6,7 @@ import { DEFAULT_ROUTING_CLASSIFIER_TIMEOUT_MS } from "../routing-classifier-con
 import { normalizeRunnerExecutionPolicy, type RunnerExecutionPolicyLike } from "../runner-execution-policy.ts";
 import { normalizeCodeReviewPolicy, type CodeReviewPolicy } from "../code-review-policy.ts";
 import { normalizeTelemetryHost, normalizeTelemetryInstallId } from "./setup-config.ts";
+import { syncTelemetryIdentityFromConfig } from "./telemetry-identity.ts";
 
 export interface ConnectedRepo {
   path: string;
@@ -169,6 +170,14 @@ export async function readSetupConfigFromHome(homePath: string): Promise<SetupCo
         // Preserve read behavior if best-effort normalization persistence fails.
       }
     }
+    try {
+      await syncTelemetryIdentityFromConfig({
+        homePath,
+        installId: normalized?.telemetry?.installId,
+      });
+    } catch {
+      // Preserve read behavior if best-effort identity sync fails.
+    }
     return normalized;
   } catch {
     return null;

src/cli/index.ts

@@ -35,6 +35,9 @@ import {
   runSetupWizard,
   type InitFlags,
 } from "./setup-wizard.ts";
+import { readOrCreateTelemetryIdentity } from "./telemetry-identity.ts";
+import { checkForPackageUpdate } from "./update-check.ts";
+import { PACKAGE_VERSION } from "../package-version.ts";
 import type { TelemetryClient } from "../telemetry.ts";
 
 interface Writer {
@@ -52,6 +55,7 @@ export interface RunCliOptions {
   providerFetch?: typeof globalThis.fetch;
   eventFetch?: typeof globalThis.fetch;
   runContextFetch?: typeof globalThis.fetch;
+  updateFetch?: typeof globalThis.fetch;
   agentRunner?: AgentRunnerHooks;
   telemetry?: CliTelemetryHook;
   writeSetupFiles?: (options: {
@@ -71,6 +75,7 @@ export async function runCli(argv: string[], options: RunCliOptions = {}): Promi
   const stderr = options.stderr ?? process.stderr;
   const detectRepo = options.detectRepoContext ?? detectRepoContext;
   const writeFiles = options.writeSetupFiles ?? writeSetupFiles;
+  const homePath = resolveAgentRailHome({ cwd, explicitHome: null });
 
   try {
     const [command, ...args] = argv;
@@ -80,6 +85,15 @@ export async function runCli(argv: string[], options: RunCliOptions = {}): Promi
       return command ? 0 : 1;
     }
 
+    await maybeRenderUpdateNotice({
+      argv,
+      homePath,
+      stderr,
+      stdoutIsTTY,
+      injectedIo: Boolean(options.stdout || options.stderr),
+      fetch: options.updateFetch,
+    });
+
     if (command === "doctor") {
       return await runDoctor(args, {
         cwd,
@@ -233,6 +247,11 @@ export async function runCli(argv: string[], options: RunCliOptions = {}): Promi
 
     const repo = await Promise.resolve(detectRepo(cwd));
     const interactiveDefault = stdinIsTTY && stdoutIsTTY && !flags.yes && !hasExplicitNonInteractiveFlags(flags);
+    const existingConfig = await readSetupConfigFromHome(homePath);
+    const telemetryIdentity = await readOrCreateTelemetryIdentity({
+      homePath,
+      existingInstallId: existingConfig?.telemetry?.installId,
+    });
 
     if (interactiveDefault || flags.interactive) {
       const prompt = (options.createPrompt ?? (() => createPromptSession()))();
@@ -241,6 +260,8 @@ export async function runCli(argv: string[], options: RunCliOptions = {}): Promi
           cwd,
           flags,
           detectedRepo: repo,
+          existingConfig,
+          telemetryInstallId: telemetryIdentity.installId,
           prompt,
           writeLine(line) {
             stdout.write(`${line}\n`);
@@ -289,6 +310,8 @@ export async function runCli(argv: string[], options: RunCliOptions = {}): Promi
       detectedRepo: repo,
       interactionMode: flags.printOnly ? "print_only" : "non_interactive",
       acceptedDefaults: flags.yes ? true : acceptedDefaultsFromFlags(flags),
+      existingConfig,
+      telemetryInstallId: telemetryIdentity.installId,
     });
 
     if (flags.yes) {
@@ -329,6 +352,39 @@ export async function runCli(argv: string[], options: RunCliOptions = {}): Promi
   }
 }
 
+async function maybeRenderUpdateNotice({
+  argv,
+  homePath,
+  stderr,
+  stdoutIsTTY,
+  injectedIo,
+  fetch,
+}: {
+  argv: string[];
+  homePath: string;
+  stderr: Writer;
+  stdoutIsTTY: boolean;
+  injectedIo: boolean;
+  fetch?: typeof globalThis.fetch;
+}): Promise<void> {
+  if (!stdoutIsTTY || argv.includes("--json") || (injectedIo && !fetch)) {
+    return;
+  }
+
+  const notice = await checkForPackageUpdate({
+    homePath,
+    currentVersion: PACKAGE_VERSION,
+    packageName: "@agentrail-core/cli",
+    fetch,
+  });
+  if (!notice) {
+    return;
+  }
+
+  stderr.write(`New AgentRail version available: ${notice.latestVersion} (current ${notice.currentVersion}).\n`);
+  stderr.write(`Run: ${notice.command}\n`);
+}
+
 function cliTelemetryAsClient(telemetry: CliTelemetryHook | undefined): TelemetryClient | undefined {
   if (!telemetry) {
     return undefined;

src/cli/setup-wizard.ts

@@ -12,7 +12,7 @@ import {
   type SetupMode,
 } from "./setup-config.ts";
 import { describeCodeReviewPolicy, normalizeCodeReviewPolicy, type CodeReviewPolicy } from "../code-review-policy.ts";
-import { resolveAgentRailHome } from "./agentrail-home.ts";
+import { resolveAgentRailHome, type SetupConfigLike } from "./agentrail-home.ts";
 import { PromptCancelledError, type PromptSession } from "./prompt.ts";
 import { normalizeRunnerExecutionPolicy, type RunnerPolicyPreset } from "../runner-execution-policy.ts";
 
@@ -52,6 +52,8 @@ export interface RunSetupWizardOptions {
   cwd: string;
   flags: InitFlags;
   detectedRepo: DetectedRepoContext;
+  existingConfig?: SetupConfigLike | null;
+  telemetryInstallId?: string;
   prompt: PromptSession;
   writeLine(line: string): void;
 }
@@ -60,6 +62,8 @@ export async function runSetupWizard({
   cwd,
   flags,
   detectedRepo,
+  existingConfig,
+  telemetryInstallId,
   prompt,
   writeLine,
 }: RunSetupWizardOptions): Promise<SetupWizardResult> {
@@ -142,7 +146,9 @@ export async function runSetupWizard({
     routingFallbackBehavior,
     codeReviewPolicy,
     runnerPolicy: normalizeRunnerExecutionPolicy({ preset: runnerPolicyPreset }),
-    telemetryEnabled: flags.telemetryEnabled,
+    telemetryEnabled: flags.telemetryEnabled ?? existingConfig?.telemetry?.enabled,
+    telemetryInstallId: telemetryInstallId ?? existingConfig?.telemetry?.installId,
+    telemetryHost: existingConfig?.telemetry?.host,
     repoPath,
     repoAllowlist,
     defaultBranch,
@@ -440,12 +446,16 @@ export function createSetupConfigFromFlags({
   detectedRepo,
   interactionMode,
   acceptedDefaults,
+  existingConfig,
+  telemetryInstallId,
 }: {
   cwd: string;
   flags: InitFlags;
   detectedRepo: DetectedRepoContext;
   interactionMode: CreateSetupConfigOptions["interactionMode"];
   acceptedDefaults: boolean;
+  existingConfig?: SetupConfigLike | null;
+  telemetryInstallId?: string;
 }): SetupConfig {
   return createSetupConfig({
     cwd,
@@ -471,6 +481,8 @@ export function createSetupConfigFromFlags({
     runnerPolicy: flags.runnerPolicyPreset
       ? normalizeRunnerExecutionPolicy({ preset: flags.runnerPolicyPreset })
       : undefined,
-    telemetryEnabled: flags.telemetryEnabled,
+    telemetryEnabled: flags.telemetryEnabled ?? existingConfig?.telemetry?.enabled,
+    telemetryInstallId: telemetryInstallId ?? existingConfig?.telemetry?.installId,
+    telemetryHost: existingConfig?.telemetry?.host,
   });
 }

src/cli/telemetry-identity.ts

@@ -0,0 +1,78 @@
+import crypto from "node:crypto";
+import path from "node:path";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+
+const TELEMETRY_IDENTITY_FILE = "telemetry-identity.json";
+const INSTALL_ID_PATTERN = /^inst_[a-f0-9]{24}$/u;
+
+export interface TelemetryIdentity {
+  installId: string;
+}
+
+export function telemetryIdentityPathForHome(homePath: string): string {
+  return path.join(homePath, TELEMETRY_IDENTITY_FILE);
+}
+
+export async function readOrCreateTelemetryIdentity({
+  homePath,
+  existingInstallId,
+}: {
+  homePath: string;
+  existingInstallId?: string | null;
+}): Promise<TelemetryIdentity> {
+  const importedInstallId = normalizeTelemetryIdentityInstallId(existingInstallId);
+  if (importedInstallId) {
+    await writeTelemetryIdentity(homePath, importedInstallId);
+    return { installId: importedInstallId };
+  }
+
+  const persistedInstallId = await readTelemetryIdentity(homePath);
+  if (persistedInstallId) {
+    return { installId: persistedInstallId };
+  }
+
+  const installId = createTelemetryIdentityInstallId();
+  await writeTelemetryIdentity(homePath, installId);
+  return { installId };
+}
+
+export async function syncTelemetryIdentityFromConfig({
+  homePath,
+  installId,
+}: {
+  homePath: string;
+  installId?: string | null;
+}): Promise<void> {
+  const normalized = normalizeTelemetryIdentityInstallId(installId);
+  if (!normalized) {
+    return;
+  }
+  await writeTelemetryIdentity(homePath, normalized);
+}
+
+function createTelemetryIdentityInstallId(): string {
+  return `inst_${crypto.randomBytes(12).toString("hex")}`;
+}
+
+function normalizeTelemetryIdentityInstallId(value: unknown): string | null {
+  return typeof value === "string" && INSTALL_ID_PATTERN.test(value) ? value : null;
+}
+
+async function readTelemetryIdentity(homePath: string): Promise<string | null> {
+  try {
+    const raw = await readFile(telemetryIdentityPathForHome(homePath), "utf8");
+    const parsed = JSON.parse(raw) as { installId?: unknown };
+    return normalizeTelemetryIdentityInstallId(parsed.installId);
+  } catch {
+    return null;
+  }
+}
+
+async function writeTelemetryIdentity(homePath: string, installId: string): Promise<void> {
+  await mkdir(homePath, { recursive: true });
+  await writeFile(
+    telemetryIdentityPathForHome(homePath),
+    `${JSON.stringify({ installId }, null, 2)}\n`,
+    "utf8",
+  );
+}