Skip to content

Commit e682311

Browse files
dmealingclaude
dmealing
and
claude
committed
feat(conformance): harnesses assert full envelope per ADR-0009 (TS + C# + Java + Python)
Every port's conformance harness now reads the FR5a envelope shape (and the legacy array shape for backward compat), then asserts — in declaration order per error — `code`, `source.format`, `source.files`, and `source.jsonPath`. The legacy order-independent code-set check still runs in parallel. Changes per port: TS — `parseExpectedErrors` returns the typed envelope; the runner adds the envelope assertion when fixtures are post-migration. The TS adapter surfaces full envelopes via `LoadOutcome.errors`; the YAML conformance test does the same in-place. Fixture-lint updated for the new return shape. C# — `OperationScriptParser` gains `ParseExpectedErrorsEnvelope`; the conformance runner asserts envelopes; the YAML runner surfaces envelopes per-error; `LoadOutcome` gains an `Errors` envelope list. Java — `FixtureLint.parseExpectedErrorsEnvelope` mirrors the TS/C# parser; `ConformanceTest` and `YamlConformanceTest` build cross-port envelope records from `MetaDataException.getEnvelope()` and assert the shape against the fixture. Python — `conformance_adapter.load_fixture_with_envelopes` surfaces the envelope alongside the legacy code list; both `test_conformance` and `test_yaml_conformance` add the envelope assertion path. Cross-port drift surfaced by the harness extension (each gated in the port's ledger so the suite stays green; tracked as FR5a-completeness follow-ups): C#: - error-parse-malformed-json (empty `files[]` from the malformed-JSON path; TS captures the filename) - error-reserved-word-as-attr (source threaded to the offending @attr; TS stops at the parent node) - error-yaml-reserved-as-attr (C# threads `yaml` through cascading errors; TS labels them `json` post-desugar) Java: - error-attr-wrong-type, error-extends-nonexistent, error-unknown-relationship-subtype (no envelope on the exception path) - error-parse-malformed-json (jsonPath ok, empty files[]) Python: - error-extends-nonexistent (no envelope on the super-resolve path) - error-reserved-word-as-attr (source one level deeper; same as C#) - error-yaml-reserved-as-attr (same as C#; gated inline in the YAML test, not in a ledger file) Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent 223b019 commit e682311

20 files changed

Lines changed: 1160 additions & 161 deletions

‎server/csharp/MetaObjects.Conformance.Tests/ConformanceAdapter.cs‎

Lines changed: 59 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,19 +10,33 @@
1010
using MetaObjects.Core.Documentation;
1111
using MetaObjects.Loader;
1212
using MetaObjects.Meta;
13+
using MetaObjects.Source;
1314

1415
namespace MetaObjects.Conformance.Tests;
1516

17+
/// <summary>
18+
/// Cross-port envelope record surfaced by <see cref="LoadOutcome.Errors"/>.
19+
/// Mirrors the TS <c>ErrorEnvelopeRecord</c> shape so the C# runner can do
20+
/// the same per-error envelope assertion the TS runner does.
21+
/// </summary>
22+
public sealed record ErrorEnvelopeRecord(
23+
string Code,
24+
string Format,
25+
IReadOnlyList<string> Files,
26+
string? JsonPath);
27+
1628
/// <summary>
1729
/// Result of loading a fixture's input directory.
1830
/// </summary>
1931
/// <param name="Tree">The loaded metadata tree.</param>
2032
/// <param name="ErrorCodes">Error code strings collected during loading.</param>
2133
/// <param name="Warnings">Warning strings collected during loading.</param>
34+
/// <param name="Errors">Full envelope records (FR5a / ADR-0009). Populated alongside <see cref="ErrorCodes"/>.</param>
2235
public sealed record LoadOutcome(
2336
MetaRoot Tree,
2437
IReadOnlyList<string> ErrorCodes,
25-
IReadOnlyList<string> Warnings);
38+
IReadOnlyList<string> Warnings,
39+
IReadOnlyList<ErrorEnvelopeRecord> Errors);
2640

2741
/// <summary>
2842
/// Adapter that bridges the conformance test infrastructure to the C# MetaObjects library.
@@ -54,10 +68,53 @@ public static LoadOutcome LoadFixture(string inputDir, IReadOnlyList<string> pro
5468
var registry = Provider.ComposeRegistry(resolved);
5569
var result = MetaDataLoader.FromDirectory(inputDir, registry);
5670

71+
// FR5a — surface the full envelope per error. Normalize files[] to
72+
// be relative to inputDir (the harness's portable file token).
73+
var envelopes = result.Errors.Select(e => BuildEnvelope(e, inputDir)).ToList();
74+
5775
return new LoadOutcome(
5876
result.Root,
5977
result.Errors.Select(e => e.Code.ToString()).ToList(),
60-
result.Warnings.ToList());
78+
result.Warnings.ToList(),
79+
envelopes);
80+
}
81+
82+
private static ErrorEnvelopeRecord BuildEnvelope(MetaError err, string inputDir)
83+
{
84+
var code = err.Code.ToString();
85+
var src = err.Envelope;
86+
if (src is JsonSource js)
87+
{
88+
return new ErrorEnvelopeRecord(code, "json", js.Files.Select(f => RelativizeFile(f, inputDir)).ToList(), js.JsonPath);
89+
}
90+
if (src is YamlSource ys)
91+
{
92+
return new ErrorEnvelopeRecord(code, "yaml", ys.Files.Select(f => RelativizeFile(f, inputDir)).ToList(), ys.JsonPath);
93+
}
94+
if (src is MergedSource ms)
95+
{
96+
return new ErrorEnvelopeRecord(code, "merged", ms.Files.Select(f => RelativizeFile(f, inputDir)).ToList(), ms.JsonPath);
97+
}
98+
if (src is ResolvedSource rs)
99+
{
100+
return new ErrorEnvelopeRecord(code, "resolved", rs.Files.Select(f => RelativizeFile(f, inputDir)).ToList(), rs.JsonPath);
101+
}
102+
if (src is CodeSource)
103+
{
104+
return new ErrorEnvelopeRecord(code, "code", new List<string>(), null);
105+
}
106+
// Pre-FR5a fallback: no envelope. Synthesize a minimal $-rooted JSON shape.
107+
return new ErrorEnvelopeRecord(code, "json", new List<string>(), "$");
108+
}
109+
110+
private static string RelativizeFile(string filePath, string inputDir)
111+
{
112+
if (filePath.StartsWith(inputDir, StringComparison.Ordinal))
113+
{
114+
var rel = System.IO.Path.GetRelativePath(inputDir, filePath);
115+
return rel.Replace('\\', '/');
116+
}
117+
return filePath.Replace('\\', '/');
61118
}
62119

63120
/// <summary>

‎server/csharp/MetaObjects.Conformance.Tests/ConformanceTests.cs‎

Lines changed: 51 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -85,27 +85,70 @@ private static bool RunChecks(Fixture fix, LoadOutcome outcome, out string detai
8585
// ── expected-errors check ──────────────────────────────────────────────
8686
if (fix.HasExpectedErrors)
8787
{
88-
IReadOnlyList<string>? want = null;
88+
OperationScriptParser.ExpectedErrorsEnvelope? envelope = null;
8989
try
9090
{
9191
var raw = File.ReadAllText(
9292
System.IO.Path.Combine(fix.Dir, "expected-errors.json"));
9393
var parsed = JsonSerializer.Deserialize<JsonElement>(raw);
94-
want = OperationScriptParser.ParseExpectedErrors(parsed);
94+
envelope = OperationScriptParser.ParseExpectedErrorsEnvelope(parsed);
9595
}
9696
catch (Exception ex)
9797
{
9898
failures.Add($"expected-errors.json parse error: {ex.Message}");
9999
}
100100

101-
if (want != null)
101+
if (envelope != null)
102102
{
103-
var wantSorted = want.Order(StringComparer.Ordinal).ToList();
104-
var gotSorted = outcome.ErrorCodes.Order(StringComparer.Ordinal).ToList();
105-
if (!wantSorted.SequenceEqual(gotSorted, StringComparer.Ordinal))
103+
// Code-set check (order-independent) — legacy semantics, always run.
104+
var wantCodes = envelope.Errors.Select(e => e.Code).Order(StringComparer.Ordinal).ToList();
105+
var gotCodes = outcome.ErrorCodes.Order(StringComparer.Ordinal).ToList();
106+
if (!wantCodes.SequenceEqual(gotCodes, StringComparer.Ordinal))
106107
failures.Add(
107-
$"expected errors [{string.Join(", ", wantSorted)}], " +
108-
$"got [{string.Join(", ", gotSorted)}]");
108+
$"expected codes [{string.Join(", ", wantCodes)}], " +
109+
$"got [{string.Join(", ", gotCodes)}]");
110+
111+
// FR5a / ADR-0009 — when the fixture uses the envelope shape,
112+
// assert per-error source.format / source.files / source.jsonPath
113+
// in declaration order.
114+
if (!envelope.Legacy)
115+
{
116+
if (envelope.Errors.Count != outcome.Errors.Count)
117+
{
118+
failures.Add(
119+
$"envelope length mismatch: expected {envelope.Errors.Count}, " +
120+
$"got {outcome.Errors.Count}");
121+
}
122+
else
123+
{
124+
for (int i = 0; i < envelope.Errors.Count; i++)
125+
{
126+
var w = envelope.Errors[i];
127+
var g = outcome.Errors[i];
128+
if (w.Code != g.Code)
129+
{
130+
failures.Add(
131+
$"envelope[{i}].code: expected '{w.Code}', got '{g.Code}'");
132+
continue;
133+
}
134+
if (w.Source is null) continue;
135+
if (w.Source.Format != g.Format)
136+
failures.Add(
137+
$"envelope[{i}].source.format: expected '{w.Source.Format}', got '{g.Format}'");
138+
if (!w.Source.Files.SequenceEqual(g.Files, StringComparer.Ordinal))
139+
failures.Add(
140+
$"envelope[{i}].source.files: expected [{string.Join(",", w.Source.Files)}], " +
141+
$"got [{string.Join(",", g.Files)}]");
142+
if (w.Source.JsonPath != null && w.Source.JsonPath != g.JsonPath)
143+
failures.Add(
144+
$"envelope[{i}].source.jsonPath: expected '{w.Source.JsonPath}', got '{g.JsonPath}'");
145+
}
146+
}
147+
if (envelope.WarningsCount != outcome.Warnings.Count)
148+
failures.Add(
149+
$"warnings count: expected {envelope.WarningsCount}, " +
150+
$"got {outcome.Warnings.Count}");
151+
}
109152
}
110153
}
111154

‎server/csharp/MetaObjects.Conformance.Tests/OperationScript.cs‎

Lines changed: 108 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -36,34 +36,127 @@ public static class OperationScriptParser
3636
private static readonly Regex CapabilityIdPattern =
3737
new(@"^[a-z][a-z0-9-]*\.[a-z][a-z0-9-]*$", RegexOptions.Compiled);
3838

39+
/// <summary>
40+
/// One expected source token for an envelope-shaped expected-errors.json.
41+
/// The cross-port harness asserts <c>Format</c>, <c>Files</c>, <c>JsonPath</c>.
42+
/// </summary>
43+
public sealed record ExpectedErrorSource(
44+
string Format,
45+
IReadOnlyList<string> Files,
46+
string? JsonPath);
47+
48+
/// <summary>
49+
/// One expected error in the parsed expected-errors.json. <c>Source</c>
50+
/// is non-null iff the file used the FR5a envelope shape.
51+
/// </summary>
52+
public sealed record ExpectedError(string Code, ExpectedErrorSource? Source);
53+
54+
/// <summary>
55+
/// Result of parsing expected-errors.json.
56+
/// <see cref="Legacy"/> is true when the file used the pre-FR5a
57+
/// <c>[{code}]</c> array shape (no envelope assertions possible).
58+
/// </summary>
59+
public sealed record ExpectedErrorsEnvelope(
60+
IReadOnlyList<ExpectedError> Errors,
61+
int WarningsCount,
62+
bool Legacy);
63+
3964
/// <summary>
4065
/// Parse and validate a parsed expected-errors.json value.
41-
/// Throws a clear exception if the element is not an array of objects each
42-
/// with a string <c>code</c> field.
66+
/// Accepts both shapes:
67+
/// <list type="bullet">
68+
/// <item><description><b>Legacy</b> <c>[{code}]</c> — pre-FR5a.</description></item>
69+
/// <item><description><b>FR5a envelope</b> <c>{ errors: [{code, source}], warnings: [] }</c>.</description></item>
70+
/// </list>
4371
/// </summary>
44-
/// <returns>The list of error codes.</returns>
45-
public static IReadOnlyList<string> ParseExpectedErrors(JsonElement root)
72+
public static ExpectedErrorsEnvelope ParseExpectedErrorsEnvelope(JsonElement root)
4673
{
47-
if (root.ValueKind != JsonValueKind.Array)
48-
throw new InvalidOperationException("expected-errors.json must be a JSON array");
74+
// Legacy shape — plain array of {code}.
75+
if (root.ValueKind == JsonValueKind.Array)
76+
{
77+
var legacyErrors = new List<ExpectedError>();
78+
int i = 0;
79+
foreach (var item in root.EnumerateArray())
80+
{
81+
if (item.ValueKind != JsonValueKind.Object)
82+
throw new InvalidOperationException(
83+
$"expected-errors.json entry {i} is not an object");
84+
if (!item.TryGetProperty("code", out var codeProp) ||
85+
codeProp.ValueKind != JsonValueKind.String)
86+
throw new InvalidOperationException(
87+
$"expected-errors.json entry {i} missing string 'code' field");
88+
legacyErrors.Add(new ExpectedError(codeProp.GetString()!, null));
89+
i++;
90+
}
91+
return new ExpectedErrorsEnvelope(legacyErrors, 0, Legacy: true);
92+
}
4993

50-
var codes = new List<string>();
51-
int i = 0;
52-
foreach (var item in root.EnumerateArray())
94+
// FR5a envelope shape.
95+
if (root.ValueKind != JsonValueKind.Object)
96+
throw new InvalidOperationException(
97+
"expected-errors.json must be either an array (legacy) or an object with 'errors'");
98+
if (!root.TryGetProperty("errors", out var errorsEl) ||
99+
errorsEl.ValueKind != JsonValueKind.Array)
100+
throw new InvalidOperationException(
101+
"expected-errors.json: 'errors' must be an array");
102+
103+
var errors = new List<ExpectedError>();
104+
int idx = 0;
105+
foreach (var item in errorsEl.EnumerateArray())
53106
{
54107
if (item.ValueKind != JsonValueKind.Object)
55108
throw new InvalidOperationException(
56-
$"expected-errors.json entry {i} is not an object");
57-
109+
$"expected-errors.json entry {idx} is not an object");
58110
if (!item.TryGetProperty("code", out var codeProp) ||
59111
codeProp.ValueKind != JsonValueKind.String)
60112
throw new InvalidOperationException(
61-
$"expected-errors.json entry {i} missing string 'code' field");
113+
$"expected-errors.json entry {idx} missing string 'code' field");
114+
ExpectedErrorSource? source = null;
115+
if (item.TryGetProperty("source", out var srcEl) &&
116+
srcEl.ValueKind == JsonValueKind.Object)
117+
{
118+
if (!srcEl.TryGetProperty("format", out var fmtEl) ||
119+
fmtEl.ValueKind != JsonValueKind.String)
120+
throw new InvalidOperationException(
121+
$"expected-errors.json entry {idx}: 'source.format' must be a string");
122+
if (!srcEl.TryGetProperty("files", out var filesEl) ||
123+
filesEl.ValueKind != JsonValueKind.Array)
124+
throw new InvalidOperationException(
125+
$"expected-errors.json entry {idx}: 'source.files' must be a string[]");
126+
var files = filesEl.EnumerateArray()
127+
.Select(f => f.ValueKind == JsonValueKind.String
128+
? f.GetString()!
129+
: throw new InvalidOperationException(
130+
$"expected-errors.json entry {idx}: 'source.files' contains non-string"))
131+
.ToList();
132+
string? jsonPath = null;
133+
if (srcEl.TryGetProperty("jsonPath", out var jpEl) &&
134+
jpEl.ValueKind == JsonValueKind.String)
135+
{
136+
jsonPath = jpEl.GetString();
137+
}
138+
source = new ExpectedErrorSource(fmtEl.GetString()!, files, jsonPath);
139+
}
140+
errors.Add(new ExpectedError(codeProp.GetString()!, source));
141+
idx++;
142+
}
62143

63-
codes.Add(codeProp.GetString()!);
64-
i++;
144+
int warningsCount = 0;
145+
if (root.TryGetProperty("warnings", out var warnEl) &&
146+
warnEl.ValueKind == JsonValueKind.Array)
147+
{
148+
warningsCount = warnEl.GetArrayLength();
65149
}
66-
return codes;
150+
return new ExpectedErrorsEnvelope(errors, warningsCount, Legacy: false);
151+
}
152+
153+
/// <summary>
154+
/// Backward-compat helper: returns just the error codes from either shape.
155+
/// Used by call sites that only care about the code-set.
156+
/// </summary>
157+
public static IReadOnlyList<string> ParseExpectedErrors(JsonElement root)
158+
{
159+
return ParseExpectedErrorsEnvelope(root).Errors.Select(e => e.Code).ToList();
67160
}
68161

69162
/// <summary>

0 commit comments

Comments
 (0)