feat(conformance): #37 Java — flattened kitchen-sink byte-match + effective-serialization gating (covers JVM incl. Kotlin)

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: dmealing

server/java/metadata/src/main/java/com/metaobjects/io/json/CanonicalJsonSerializer.java

@@ -287,42 +287,38 @@ private static JsonObject serializeBody(MetaData node, boolean effective, String
             body.addProperty(KEY_NAME, shortName);
         }
 
-        // 2. package — emit only where introduced (differs from parent's package).
+        // 2. package — emit only where the TS oracle sets `model.package`.
         //
         // Special case: MetaRoot's full name IS the package (e.g. "acme::commerce").
         // For all other nodes, getPackage() gives the package prefix of the qualified name.
         //
-        // WHY "differs from parent" rather than a naive "emit if non-empty":
-        // The TS oracle uses model.package, which is the *authored* package — undefined
-        // when the author did not write a package key, so "emit if present" works there.
-        // Java's getPackage() returns the *effective* (inherited) package: every node in
-        // an "acme::commerce" tree would return "acme::commerce" even when they never
-        // authored it. Emitting unconditionally would wrongly inject a "package" key on
-        // every node. Emitting only when the value differs from the parent's resolved
-        // package approximates "authored here for the first time."
+        // The TS oracle emits `package` iff `model.package` is set, which happens for:
+        //   (a) the root node (its package is always present);
+        //   (b) a node that explicitly authored a `package` key on its body;
+        //   (c) a MetaField whose parent is NOT a MetaObject (field-at-root /
+        //       abstract shared field) — TS's package-inheritance rule sets
+        //       `model.package` from the file context for these.
         //
-        // Known imprecision (Task 2 follow-up): a child that redundantly re-authors the
-        // exact same package as its parent would be suppressed here but emitted by TS.
-        // Tracking the truly-authored package requires a separate flag on MetaData that
-        // does not exist yet; that is the Task 2 work.
+        // We must NOT use "differs from parent package" as a trigger: a single
+        // loader builds ONE MetaRoot for a multi-file load, so children merged
+        // from a file with a different package would spuriously emit a `package`
+        // key that TS never emits (it threads authored-package, not effective
+        // package). Effective package is recovered structurally on reload from
+        // the parent context, so suppressing it here is byte-correct and
+        // round-trips.
         String nodePackage = resolveNodePackage(node);
         if (nodePackage != null && !nodePackage.isEmpty()) {
-            boolean differsFromParent = (parentPackage == null) || !nodePackage.equals(parentPackage);
-            // Cross-port: abstract field-type nodes declared at root level
-            // (e.g. an abstract field.enum bound as a shared type) always emit
-            // their package, even when it equals the root's package — they are
-            // addressable library entries that need a stable qualifier so other
-            // entities can reference them via `extends`. Objects / templates /
-            // layouts at root do not need the redundant emission.
-            boolean rootAbstractFieldType = (node.getParent() instanceof MetaRoot)
-                && (node instanceof com.metaobjects.field.MetaField)
-                && getIsAbstractValue(node);
+            boolean isRoot = node instanceof MetaRoot;
             // Cross-port byte-parity: when the author explicitly wrote a
-            // `package` key on this node's body (even one that happens to
-            // equal the parent's package), round-trip it on the way out.
+            // `package` key on this node's body, round-trip it on the way out.
             // CanonicalJsonParser tracks this via MetaData.isPackageAuthored().
             boolean explicitlyAuthored = node.isPackageAuthored();
-            if (differsFromParent || rootAbstractFieldType || explicitlyAuthored) {
+            // TS package-inheritance rule (c): a MetaField NOT directly inside a
+            // MetaObject (declared at root, or in another non-object container)
+            // carries its own package so it stays addressable via `extends`.
+            boolean fieldOutsideObject = (node instanceof com.metaobjects.field.MetaField)
+                && !(node.getParent() instanceof com.metaobjects.object.MetaObject);
+            if (isRoot || explicitlyAuthored || fieldOutsideObject) {
                 body.addProperty(KEY_PACKAGE, nodePackage);
             }
         }
@@ -438,47 +434,111 @@ private static List<MetaData> collectOwnStructuralChildren(MetaData node) {
     }
 
     /**
-     * Returns structural children (own + inherited from super chain) de-duplicated by
-     * type+name, preserving own children first.
+     * Returns the effective structural children of {@code node} (own + inherited
+     * via the super chain), in the cross-port canonical order.
+     *
+     * <p>Mirrors the TS oracle's {@code _effectiveChildren} exactly: start from
+     * the super's effective children, then for each own child either OVERRIDE
+     * the matching (type, name) super child in place (preserving the super's
+     * position) or APPEND it at the end when it introduces a new member. This
+     * places inherited members first, with newly-introduced own members last —
+     * the byte-order the {@code expected-effective.json} fixtures assert.</p>
+     *
+     * <p>Java's {@code getChildren(MetaData.class, true)} de-duplicates by
+     * type+name but lists own children first, which diverges from the oracle —
+     * hence this explicit recursive merge.</p>
      */
     private static List<MetaData> collectEffectiveStructuralChildren(MetaData node) {
-        // getChildren(MetaData.class, true) walks the super chain but de-duplicates by type-name key.
-        List<MetaData> all = node.getChildren(MetaData.class, true);
-        List<MetaData> result = new ArrayList<>();
-        for (MetaData child : all) {
-            if (!(child instanceof MetaAttribute)) {
-                result.add(child);
+        return effectiveChildrenMerged(node, new java.util.IdentityHashMap<>());
+    }
+
+    private static List<MetaData> effectiveChildrenMerged(MetaData node,
+                                                          java.util.Map<MetaData, Boolean> visited) {
+        MetaData superData = node.hasSuperData() ? node.getSuperData() : null;
+
+        if (superData == null || visited.containsKey(superData)) {
+            return collectOwnStructuralChildren(node);
+        }
+        visited.put(superData, Boolean.TRUE);
+
+        // Start from the super's effective children (a fresh, mutable copy).
+        List<MetaData> result = new ArrayList<>(effectiveChildrenMerged(superData, visited));
+
+        List<MetaData> appendQueue = new ArrayList<>();
+        for (MetaData ownChild : collectOwnStructuralChildren(node)) {
+            int idx = -1;
+            for (int i = 0; i < result.size(); i++) {
+                MetaData sc = result.get(i);
+                if (sc.getType().equals(ownChild.getType())
+                        && canonicalMatchName(sc).equals(canonicalMatchName(ownChild))) {
+                    idx = i;
+                    break;
+                }
+            }
+            if (idx != -1) {
+                result.set(idx, ownChild); // in-place override, super's position kept
+            } else {
+                appendQueue.add(ownChild);
             }
         }
+        result.addAll(appendQueue);
         return result;
     }
 
+    /**
+     * The name used for cross-port effective-override matching. Mirrors the TS
+     * oracle, whose auto-named nodes carry {@code name === ""}: an auto-generated
+     * name (e.g. a {@code source.rdb}'s {@code rdb1}) collapses to the empty
+     * string so an own auto-named node overrides its inherited counterpart at the
+     * super's position — exactly as TS matches {@code (type, "")} against
+     * {@code (type, "")}. Java assigns real {@code rdbN} auto-names (which the
+     * serializer suppresses), so without this collapse the per-package counter
+     * would make Product's {@code rdb1} and ProductSummary's {@code rdb2} fail to
+     * match and the inherited source would wrongly duplicate.
+     */
+    private static String canonicalMatchName(MetaData node) {
+        if (isAutoGeneratedName(node)) {
+            return "";
+        }
+        String s = node.getShortName();
+        return s == null ? "" : s;
+    }
+
     // ---------------------------------------------------------------------------
     // Helpers — package resolution
     // ---------------------------------------------------------------------------
 
     /**
      * Returns the "authoring package" of a node — the package context an author
-     * would use to write its {@code extends} ref. For root-level nodes this is
-     * the root's own package; for child nodes it walks up to the nearest
-     * MetaObject (entity / value) and returns that ancestor's package. Used by
-     * extends-ref serialization so a field inside an entity in package X
-     * references a sibling in X by short name (not by FQN).
+     * would use to write its {@code extends} ref. For a node that is itself a
+     * top-level type (a child of the MetaRoot — an object / template / abstract
+     * field), this is the node's OWN package. For a node nested inside an entity
+     * it walks up to the nearest MetaObject and returns that ancestor's package.
+     * Used by extends-ref serialization so a sibling in the same package is
+     * referenced by short name (not by FQN).
+     *
+     * <p>Cross-port correctness for the flattened multi-file case: a single
+     * loader builds ONE MetaRoot whose name is just the first input file's
+     * package. The other files contribute children in OTHER packages. The
+     * root's own package is therefore NOT the authoring context for those
+     * children — each node's own {@link MetaData#getPackage()} (the effective
+     * package of its FQN) is. So we never fall back to the root's package; we
+     * fall back to the node's own package.</p>
      */
     private static String authoringPackageFor(MetaData node) {
         MetaData current = node;
         while (current != null) {
-            if (current instanceof MetaRoot) {
-                return resolveNodePackage(current);
-            }
             MetaData parent = current.getParent();
-            if (parent == null) break;
+            if (parent == null || parent instanceof MetaRoot) {
+                // `current` is a top-level type — its own package IS the
+                // authoring context (not the shared root's package).
+                return current.getPackage();
+            }
             if (parent instanceof com.metaobjects.object.MetaObject) {
                 // current is a child of an entity → authoring context = entity's package
                 return parent.getPackage();
             }
-            // Non-MetaObject parent (or MetaRoot): keep walking up; MetaRoot is
-            // caught on the next iteration's top-of-loop check.
+            // Non-MetaObject parent: keep walking up.
             current = parent;
         }
         return node.getPackage();

server/java/metadata/src/test/java/com/metaobjects/conformance/ConformanceTest.java

@@ -78,9 +78,11 @@
  *       so canonical round-trips produce the right top-level {@code package}.</li>
  *   <li>Warnings: the Java loader has no warning surface yet. Fixtures with
  *       {@code expected-warnings.json} are ledgered as gaps.</li>
- *   <li>Effective serialization + script execution: not implemented in the
- *       Java harness yet. Fixtures using {@code expected-effective.json} or
- *       {@code script.json} are ledgered.</li>
+ *   <li>Effective serialization: supported. A fixture's
+ *       {@code expected-effective.json} (extends RESOLVED — inherited members
+ *       inlined) is byte-compared against
+ *       {@link CanonicalJsonSerializer#canonicalSerializeEffective(MetaData)}
+ *       when present.</li>
  * </ul>
  */
 @RunWith(Parameterized.class)
@@ -271,9 +273,9 @@ private static void runConformanceChecks(FixtureDiscovery.Fixture fix,
                 }
             }
         }
-        if (fix.hasExpectedEffective) {
-            failures.add("expected-effective.json (effective serialization) not supported by Java harness");
-        }
+        // expected-effective.json is now supported — the assertion lives below,
+        // after the loader has run and built the tree (so the effective
+        // serialization can resolve the extends chain).
         // expected-warnings.json is now supported — the assertion lives below,
         // after the loader has run (so loader.getWarnings() has a value to
         // compare against).
@@ -447,6 +449,26 @@ private static void runConformanceChecks(FixtureDiscovery.Fixture fix,
             }
         }
 
+        // -- expected-effective.json check ----------------------------------
+        // Mirrors the TS runner: when a fixture ships expected-effective.json,
+        // emit the EFFECTIVE canonical serialization (extends resolved —
+        // inherited members inlined) and byte-compare (newline-normalized).
+        if (fix.hasExpectedEffective) {
+            String want;
+            try {
+                want = new String(Files.readAllBytes(fix.dir.resolve("expected-effective.json")),
+                    StandardCharsets.UTF_8).trim();
+            } catch (IOException ex) {
+                failures.add("expected-effective.json read error: " + ex.getMessage());
+                return;
+            }
+            String got = CanonicalJsonSerializer.canonicalSerializeEffective(loader.getRoot()).trim();
+            if (!want.equals(got)) {
+                failures.add("effective serialization mismatch:\n--- expected ---\n"
+                    + want + "\n--- got ---\n" + got);
+            }
+        }
+
         // -- expected-warnings check ----------------------------------------
         // Match the TS/C# contract: warnings are compared as a multiset of exact
         // strings (order-insensitive). If a fixture has NO expected-warnings.json

server/java/metadata/src/test/java/com/metaobjects/io/json/CanonicalJsonSerializerTest.java

@@ -267,10 +267,18 @@ public void testCanonicalSerializeEffectiveIncludesInheritedAttrs() {
     }
 
     // -----------------------------------------------------------------------
-    // Test 10 — child in a different package emits a "package" key
+    // Test 10 — package emission follows the cross-port AUTHORED-package rule
+    //
+    // The TS oracle emits a node's `package` key iff the author wrote one on the
+    // node body (or it is the root / a field outside an object). A child object
+    // whose effective package merely DIFFERS from its parent's does NOT emit a
+    // redundant `package` key — the package is recovered structurally on reload.
+    // This is the byte-correct cross-port behavior (a single loader builds ONE
+    // MetaRoot for a multi-file load, so children merged from a file with a
+    // different package must not spuriously emit a key TS never emits).
     // -----------------------------------------------------------------------
     @Test
-    public void testChildInDifferentPackageEmitsPackageKey() {
+    public void testChildInDifferentPackageDoesNotEmitPackageKeyUnlessAuthored() {
         // Root is "acme::core"; child object lives in "acme::commerce" (different package).
         MetaRoot root = new MetaRoot("acme::core");
 
@@ -279,9 +287,16 @@ public void testChildInDifferentPackageEmitsPackageKey() {
 
         String json = CanonicalJsonSerializer.canonicalSerialize(root);
 
-        // The child node must emit its own "package" key because it differs from the root's.
-        assertTrue("child node must emit 'package' key when it differs from parent's, got: " + json,
+        // Effective-package difference alone must NOT emit a `package` key.
+        assertFalse("unauthored child must NOT emit a redundant 'package' key, got: " + json,
                 json.contains("\"package\": \"acme::commerce\""));
+
+        // When the author explicitly wrote the package (parser sets this flag),
+        // it round-trips on the way out.
+        product.setPackageAuthored(true);
+        String authoredJson = CanonicalJsonSerializer.canonicalSerialize(root);
+        assertTrue("authored child must emit its 'package' key, got: " + authoredJson,
+                authoredJson.contains("\"package\": \"acme::commerce\""));
     }
 
     // -----------------------------------------------------------------------