fix: add structural noise filtering for system traces and raw blobs

src/noise-filter.ts

@@ -50,19 +50,34 @@ const DIAGNOSTIC_ARTIFACT_PATTERNS = [
   /\bno explicit solution\b/i,
 ];
 
+// Structural noise: runtime traces, raw blobs, wrappers, and malformed fragments
+const STRUCTURAL_NOISE_PATTERNS = [
+  /^System\s*:/i,
+  /compacti(ng|on)\s*(context|safeguard)/i,
+  /model\s*(switch|switched|changed|swap)/i,
+  /session\s*(reset|restart|start|end)/i,
+  /\(untrusted metadata\)\s*:/i,
+  /^\{[\s\S]*\}$/,
+  /<[a-z-]+>[\s\S]*<\/[a-z-]+>/i,
+  /(?:^>.*\n){3,}/m,
+];
+
 export interface NoiseFilterOptions {
   /** Filter agent denial responses (default: true) */
   filterDenials?: boolean;
   /** Filter meta-questions about memory (default: true) */
   filterMetaQuestions?: boolean;
   /** Filter session boilerplate (default: true) */
   filterBoilerplate?: boolean;
+  /** Filter structural noise (default: true) */
+  filterStructuralNoise?: boolean;
 }
 
 const DEFAULT_OPTIONS: Required<NoiseFilterOptions> = {
   filterDenials: true,
   filterMetaQuestions: true,
   filterBoilerplate: true,
+  filterStructuralNoise: true,
 };
 
 /**
@@ -78,6 +93,7 @@ export function isNoise(text: string, options: NoiseFilterOptions = {}): boolean
   if (opts.filterDenials && DENIAL_PATTERNS.some(p => p.test(trimmed))) return true;
   if (opts.filterMetaQuestions && META_QUESTION_PATTERNS.some(p => p.test(trimmed))) return true;
   if (opts.filterBoilerplate && BOILERPLATE_PATTERNS.some(p => p.test(trimmed))) return true;
+  if (opts.filterStructuralNoise && STRUCTURAL_NOISE_PATTERNS.some(p => p.test(trimmed))) return true;
   if (DIAGNOSTIC_ARTIFACT_PATTERNS.some(p => p.test(trimmed))) return true;
 
   return false;

test/noise-filter-structural.test.mjs

@@ -0,0 +1,32 @@
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import jitiFactory from "jiti";
+
+const jiti = jitiFactory(import.meta.url, { interopDefault: true });
+
+const { isNoise } = jiti("../src/noise-filter.ts");
+
+describe("noise-filter structural patterns", () => {
+  const samples = [
+    "System: compaction safeguard engaged",
+    "Compaction context safeguard tripped",
+    "model switch detected",
+    "model changed to gpt-5",
+    "session reset due to inactivity",
+    "(untrusted metadata): Sender (untrusted metadata): foo",
+    "{\"type\":\"meta\",\"payload\":{\"note\":\"wrapper\"}}",
+    "<relevant-memories>foo</relevant-memories>",
+    "> quote one\n> quote two\n> quote three\n> quote four\n",
+  ];
+
+  for (const input of samples) {
+    it(`filters structural noise: ${input.slice(0, 40)}`, () => {
+      assert.equal(isNoise(input), true);
+    });
+  }
+
+  it("allows structural noise when disabled", () => {
+    const input = "System: model switch detected";
+    assert.equal(isNoise(input, { filterStructuralNoise: false }), false);
+  });
+});