Merge worktree-fr5b-ts-2026-05-26 — FR5b TS reference port (YAML source positions; format stays json for cross-port-safety until C#/Java/Python land FR5b)

Author: dmealing

server/typescript/packages/metadata/src/core/parser-yaml.ts

@@ -1,19 +1,31 @@
 // Authoring YAML parser.
 //
-// parseYaml is a front-end: yaml.parse → desugar → the shared buildTree
-// (parser-core.ts). The desugar applies the four authoring-sugar rules so the
-// resulting typed tree is identical to the one the equivalent canonical JSON
-// produces.
+// parseYaml is a front-end: parseYamlWithPositions → desugar → the shared
+// buildTree (parser-core.ts). The desugar applies the four authoring-sugar
+// rules so the resulting typed tree is identical to the one the equivalent
+// canonical JSON produces.
+//
+// FR5b: the parse phase now preserves YAML source positions through the
+// pipeline. parseYamlWithPositions attaches a Symbol-keyed position-by-key
+// map onto every mapping object; desugar shallow-copies that property; and
+// buildTree's `format: "yaml"` mode reads the position when stamping
+// `node.source.yamlPosition` (see parser-core.ts).
 
-import { parse as parseYamlText } from "yaml";
 import { ParseError } from "../errors.js";
 import { buildTree } from "../parser-core.js";
 import type { ParseOptions, ParseResult } from "../parser-core.js";
 import type { ErrorSource } from "../source.js";
 import { desugar } from "./yaml-desugar.js";
+import { parseYamlWithPositions } from "./yaml-positions-walker.js";
 
 /** FR5a / ADR-0009 — build a YAML-source envelope rooted at "$".
- *  yamlPosition is FR5b territory and not populated here. */
+ *  yamlPosition on the envelope is left undefined here; envelopes for
+ *  THROWN parser errors carry positions only when the parse phase pushed
+ *  the path into the live parser-core builder (see populateNodeSource in
+ *  parser-core.ts). Errors raised before buildTree (e.g. yaml syntax
+ *  failures, an empty desugar result) lack a node — `yamlPosition` is
+ *  intentionally omitted, per the spec's "skip on desugar-synthesized
+ *  nodes" decision. */
 function yamlSource(sourceName: string | undefined): ErrorSource {
   return {
     format: "yaml",
@@ -31,7 +43,7 @@ export function parseYaml(content: string, opts: ParseOptions): ParseResult {
   // The loader's per-source try/catch collects the throw into LoadResult.errors.
   let parsed: unknown;
   try {
-    parsed = parseYamlText(normalizedContent);
+    parsed = parseYamlWithPositions(normalizedContent).value;
   } catch (err) {
     throw new ParseError(
       `Invalid YAML: ${(err as Error).message}`,
@@ -52,7 +64,10 @@ export function parseYaml(content: string, opts: ParseOptions): ParseResult {
     );
   }
 
-  const result = buildTree(canonical, opts);
+  // FR5b — buildTree needs to know the source-format discriminant so
+  // populateNodeSource emits `format: "yaml"` envelopes (with the
+  // optional yamlPosition) instead of the default `format: "json"`.
+  const result = buildTree(canonical, { ...opts, sourceFormat: "yaml" });
 
   // Merge collected desugar errors ahead of buildTree's own collected errors.
   // Each CollectedError carries its own stable code when set (e.g.

server/typescript/packages/metadata/src/core/yaml-desugar.ts

@@ -38,6 +38,11 @@ import {
   RESERVED_KEY_IS_ARRAY,
   TYPE_SUBTYPE_SEPARATOR,
 } from "../shared/structural.js";
+import {
+  getPositionMap,
+  setPositionMap,
+  type PositionMap,
+} from "./yaml-positions.js";
 import {
   ATTR_SUBTYPE_STRING,
   ATTR_SUBTYPE_CLASS,
@@ -99,6 +104,11 @@ function desugarNode(
   const rawKey = entries[0]!;
   const rawBody = (input as Record<string, unknown>)[rawKey];
 
+  // FR5b — capture the wrapper-level position-by-key map BEFORE re-keying.
+  // The author's raw key (with `[]` suffix and possibly omitted subType) is
+  // the lookup key; the desugar's canonical key is what we emit.
+  const wrapperPositions = getPositionMap(input);
+
   // Rule 4: a trailing "[]" on the key → isArray.
   let key = rawKey;
   let isArray = false;
@@ -111,7 +121,10 @@ function desugarNode(
   const canonicalKey = resolveKey(key, registry, errors, path);
 
   // Rule 2: a scalar body → { name: <scalar> }.
-  const body = desugarBody(rawBody, registry, canonicalKey, errors, path);
+  // FR5b — propagate the wrapper-key's position into the synthesized body
+  // when the input body was a scalar (no body-side positions to inherit).
+  const wrapperKeyPos = wrapperPositions?.[rawKey];
+  const body = desugarBody(rawBody, registry, canonicalKey, errors, path, wrapperKeyPos);
 
   // Rule 4 (cont.): stamp isArray onto the canonical body.
   if (isArray) body[RESERVED_KEY_IS_ARRAY] = true;
@@ -131,7 +144,16 @@ function desugarNode(
   }
   // A non-array `children` value is left untouched — buildTree reports it.
 
-  return { [canonicalKey]: body };
+  // FR5b — emit a wrapper-level position-by-key map for the canonical wrapper
+  // so buildTree's per-child iteration can read the position via the same
+  // lookup it uses for JSON input. The single key transformation is
+  // rawKey → canonicalKey (Rule 1 fuses the subType, Rule 4 strips `[]`).
+  const outWrapper: Record<string, unknown> = { [canonicalKey]: body };
+  if (wrapperKeyPos !== undefined) {
+    setPositionMap(outWrapper, { [canonicalKey]: wrapperKeyPos });
+  }
+
+  return outWrapper;
 }
 
 // Rule 1 — resolve a possibly-bare key to a fused `type.subType` token.
@@ -169,16 +191,30 @@ function desugarBody(
   canonicalKey: string,
   errors: CollectedError[],
   path: string,
+  /** FR5b — position of the WRAPPER key (the `field.string:` line). Used to
+   *  back-fill `yamlPosition` on synthesized bodies (Rule 2's scalar lift) +
+   *  empty bodies; for mapping bodies we use the body's own position-by-key
+   *  map. */
+  wrapperKeyPos: { line: number; col: number } | undefined,
 ): Record<string, unknown> {
   if (
     typeof rawBody === "string" ||
     typeof rawBody === "number" ||
     typeof rawBody === "boolean"
   ) {
-    return { [RESERVED_KEY_NAME]: rawBody };
+    // FR5b — the synthesized `{ name: rawBody }` has no YAML-side
+    // counterpart; we attribute the `name` slot to the wrapper-key's
+    // position (the only YAML position that meaningfully belongs to this
+    // synthesis).
+    const out: Record<string, unknown> = { [RESERVED_KEY_NAME]: rawBody };
+    if (wrapperKeyPos !== undefined) {
+      setPositionMap(out, { [RESERVED_KEY_NAME]: wrapperKeyPos });
+    }
+    return out;
   }
   if (rawBody === null || rawBody === undefined) {
     // An empty body (`field.string:` with nothing after) → an empty node.
+    // No body keys to position; the wrapper carries the node's position.
     return {};
   }
   if (Array.isArray(rawBody)) {
@@ -192,20 +228,38 @@ function desugarBody(
   // Rule D2 (type-coercion guard).
   const src = rawBody as Record<string, unknown>;
   const out: Record<string, unknown> = {};
+  // FR5b — translate the body's position-by-key map across the sigil-free
+  // rewrite. A bare `filterable` key in the source maps to `@filterable` in
+  // the canonical body; the YAML position belongs to BOTH names (the YAML
+  // author only wrote one). We re-key the position map to match the canonical
+  // body's keys so buildTree's per-attr inspection (FR5b follow-ups, e.g.
+  // ERR_BAD_ATTR_VALUE) can find the position via the canonical key.
+  const srcPositions = getPositionMap(src);
+  const outPositions: PositionMap = {};
+  let hasOutPositions = false;
   const schemaIndex = attrSchemaIndex(registry, canonicalKey);
   for (const key of Object.keys(src)) {
+    let outKey: string;
     if (RESERVED_KEYS.has(key) || key.startsWith(ATTR_PREFIX)) {
       out[key] = src[key];
+      outKey = key;
       // D2 also applies to author-written @-keys (the awkward form).
       const attrName = key.startsWith(ATTR_PREFIX) ? key.slice(ATTR_PREFIX.length) : "";
       if (attrName !== "" && !RESERVED_KEYS.has(attrName)) {
         checkCoercion(attrName, src[key], schemaIndex, errors, path);
       }
     } else {
-      out[`${ATTR_PREFIX}${key}`] = src[key];
+      outKey = `${ATTR_PREFIX}${key}`;
+      out[outKey] = src[key];
       checkCoercion(key, src[key], schemaIndex, errors, path);
     }
+    const pos = srcPositions?.[key];
+    if (pos !== undefined) {
+      outPositions[outKey] = pos;
+      hasOutPositions = true;
+    }
   }
+  if (hasOutPositions) setPositionMap(out, outPositions);
   return out;
 }
 

server/typescript/packages/metadata/src/core/yaml-positions-walker.ts

@@ -0,0 +1,101 @@
+// FR5b — YAML AST → JS walker that preserves source positions.
+//
+// This module is the only place inside @metaobjectsdev/metadata that
+// imports the `yaml` package. It lives in core/ alongside parser-yaml.ts,
+// and is reached only via that parser — never via src/index.ts. The
+// browser-safety test guards this invariant.
+
+import {
+  parseDocument,
+  isAlias,
+  isMap,
+  isScalar,
+  isSeq,
+  LineCounter,
+  type Document,
+} from "yaml";
+
+import {
+  setPositionMap,
+  type PositionMap,
+} from "./yaml-positions.js";
+
+/** Result of parsing YAML text with positions retained. */
+export interface YamlParseResult {
+  /** The JS object (same shape as `yaml.parse(text)` returns), with
+   *  position-by-key maps attached to every mapping. */
+  value: unknown;
+  /** The yaml library's LineCounter — exposed for callers that need to map
+   *  additional ranges (e.g. surfacing errors raised by the YAML library
+   *  itself). */
+  lineCounter: LineCounter;
+}
+
+/** Parse YAML text and return a JS object with positions attached.
+ *
+ *  Mirrors the contract of `yaml.parse(text)` for the shapes the metaobjects
+ *  authoring grammar uses (mappings, sequences, scalars). Aliases and tags
+ *  are deferred via the underlying parseDocument call — i.e. they resolve as
+ *  the library normally would.
+ *
+ *  Throws on YAML syntax errors (same behavior as `yaml.parse`). */
+export function parseYamlWithPositions(text: string): YamlParseResult {
+  const lineCounter = new LineCounter();
+  const doc = parseDocument(text, { lineCounter });
+  // Surface YAML syntax errors as a throw, matching `yaml.parse` behavior.
+  // (parseDocument collects them rather than throwing.)
+  if (doc.errors.length > 0) {
+    throw doc.errors[0]!;
+  }
+  const value = yamlNodeToJs(doc.contents, lineCounter, doc);
+  return { value, lineCounter };
+}
+
+// Walk a yaml AST node into a JS structure. For each YAMLMap, attach a
+// position-by-key map onto the resulting JS object — the position of each
+// key is the (line, col) of the KEY token in the YAML source.
+function yamlNodeToJs(
+  node: unknown,
+  lineCounter: LineCounter,
+  doc: Document,
+): unknown {
+  if (node === null || node === undefined) return null;
+  if (isScalar(node)) {
+    // Honour the library's default scalar typing (numbers / booleans /
+    // strings / null all come through Scalar.value).
+    return node.value;
+  }
+  if (isAlias(node)) {
+    // Resolve an anchor alias (e.g. `*col` after `&col sku_code`) to its
+    // target value — same behaviour as the library's toJS().
+    const target = node.resolve(doc);
+    return yamlNodeToJs(target, lineCounter, doc);
+  }
+  if (isMap(node)) {
+    const out: Record<string, unknown> = {};
+    const positions: PositionMap = {};
+    let hasAnyPosition = false;
+    for (const pair of node.items) {
+      // Only string-keyed entries are valid in metaobjects authoring; ignore
+      // exotic keys (numeric / complex) — they'd already break the desugar.
+      if (!isScalar(pair.key)) continue;
+      const keyText = String(pair.key.value);
+      const valueJs = yamlNodeToJs(pair.value, lineCounter, doc);
+      out[keyText] = valueJs;
+      const keyRange = pair.key.range;
+      if (keyRange !== null && keyRange !== undefined) {
+        const pos = lineCounter.linePos(keyRange[0]);
+        positions[keyText] = { line: pos.line, col: pos.col };
+        hasAnyPosition = true;
+      }
+    }
+    if (hasAnyPosition) setPositionMap(out, positions);
+    return out;
+  }
+  if (isSeq(node)) {
+    return node.items.map((item) => yamlNodeToJs(item, lineCounter, doc));
+  }
+  // Tags / unsupported — fall back to null. The metaobjects authoring
+  // grammar does not use them.
+  return null;
+}

server/typescript/packages/metadata/src/core/yaml-positions.ts

@@ -0,0 +1,80 @@
+// FR5b — YAML authoring source-position carrier (per ADR-0009).
+//
+// This module is split into two layers so the browser bundle (which
+// imports from src/index.ts) stays free of the Node-only `yaml` package:
+//
+//   - yaml-positions.ts (this file) — pure types + Symbol + accessors. NO
+//     `yaml` import. Imported by parser-core.ts so the source-on-node
+//     stamper can read positions when stamping `format: "yaml"` envelopes.
+//   - yaml-positions-walker.ts — depends on `yaml`. Imported only by
+//     parser-yaml.ts (and parser-yaml itself only ships server-side).
+//
+// Source-map carrier (per the FR5b spec's "open question" §2): a
+// Symbol-keyed, non-enumerable property on the wrapper-mapping object. The
+// symbol is the well-known cross-port key
+// `Symbol.for("@metaobjectsdev/yamlPositionByKey")`, so any plugin that
+// touches the canonical JS can read positions if it knows to look. The
+// map's keys are the wrapper's own keys (e.g. "object.entity" for a
+// wrapper `{ "object.entity": { ... } }` or "name" / "package" /
+// "children" for the body keys of a node).
+//
+// Rationale for "symbol-keyed property" over a parallel sourcemap / wrapper
+// type:
+//   - Invisible to JSON.stringify and Object.keys (non-enumerable).
+//   - No parallel data structure to keep in sync — the position rides with
+//     the node it describes.
+//   - No wrapper type — desugar still operates on plain JS objects, so the
+//     existing Rule 1–5 logic does not need a rewrite.
+//
+// On desugar-synthesized nodes (Rule 2's scalar-body lift): the synthesized
+// body `{ name: rawScalar }` inherits the wrapper key's position from the
+// parent's position map. On any other synthesis (Rule 4's isArray stamping,
+// for example), the position survives because we shallow-copy via the
+// existing desugar path.
+
+/** Cross-port well-known symbol key for the position-by-key map. */
+export const YAML_POSITION_BY_KEY = Symbol.for(
+  "@metaobjectsdev/yamlPositionByKey",
+);
+
+/** A YAML source position — 1-indexed line and column. */
+export interface YamlPosition {
+  readonly line: number;
+  readonly col: number;
+}
+
+/** The position-by-key map attached to a mapping object. */
+export type PositionMap = Record<string, YamlPosition>;
+
+/** Read the position-by-key map from a JS object, if present.
+ *  Returns undefined for primitives, arrays, null, and untagged objects. */
+export function getPositionMap(obj: unknown): PositionMap | undefined {
+  if (obj === null || typeof obj !== "object" || Array.isArray(obj)) {
+    return undefined;
+  }
+  return (obj as { [YAML_POSITION_BY_KEY]?: PositionMap })[YAML_POSITION_BY_KEY];
+}
+
+/** Read the position for a specific key on a mapping object. */
+export function getYamlPosition(
+  obj: unknown,
+  key: string,
+): YamlPosition | undefined {
+  const map = getPositionMap(obj);
+  return map?.[key];
+}
+
+/** Attach (or replace) the position-by-key map on a mapping object. The map
+ *  property is non-enumerable so JSON.stringify and `for (const k in obj)`
+ *  loops do not see it. */
+export function setPositionMap(
+  obj: Record<string, unknown>,
+  positions: PositionMap,
+): void {
+  Object.defineProperty(obj, YAML_POSITION_BY_KEY, {
+    value: positions,
+    enumerable: false,
+    writable: true,
+    configurable: true,
+  });
+}