diff --git a/.github/workflows/compatibility.yml b/.github/workflows/compatibility.yml index d89cf93..2aa8a86 100644 --- a/.github/workflows/compatibility.yml +++ b/.github/workflows/compatibility.yml @@ -9,6 +9,22 @@ on: pull_request: jobs: + tests: + name: test suite + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + - name: Prepare local package feed + run: | + mkdir -p artifacts/packages + dotnet pack src/AstFirst.Core/AstFirst.Core.csproj -c Release -o artifacts/packages + dotnet pack src/AstFirst.Runtime/AstFirst.Runtime.csproj -c Release -o artifacts/packages + - name: Run tests + run: dotnet test AstFirst.slnx -c Release --nologo + compat: name: ${{ matrix.sdk }} / ${{ matrix.framework }} strategy: diff --git a/.github/workflows/publish-nuget.yml b/.github/workflows/publish-nuget.yml index 2996291..2f4cc01 100644 --- a/.github/workflows/publish-nuget.yml +++ b/.github/workflows/publish-nuget.yml @@ -24,6 +24,8 @@ jobs: fetch-depth: 0 # タグ存在確認のため全履歴＋タグを取得 - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' - name: Version を読み取り id: ver @@ -71,6 +73,10 @@ jobs: if: steps.check.outputs.published == 'false' run: dotnet pack src/AstFirst.Runtime/AstFirst.Runtime.csproj -c Release -p:Version=${{ steps.ver.outputs.version }} -p:PackageReadmePath="$PWD/README.nuget.md" -o artifacts/packages + - name: Test + if: steps.check.outputs.published == 'false' + run: dotnet test AstFirst.slnx -c Release --nologo + - name: Pack Generator if: steps.check.outputs.published == 'false' run: dotnet pack src/AstFirst.Generator/AstFirst.Generator.csproj -c Release -p:Version=${{ steps.ver.outputs.version }} -p:PackageReadmePath="$PWD/README.nuget.md" -o artifacts/packages diff --git a/CHANGELOG.md b/CHANGELOG.md index dc005c1..4f26601 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## 0.4.1 — 文法ノード探索の拡張と品質改善 + +- **別名前空間の文法ノード**: 既定探索で `[Grammar]` ルートの派生型をアセンブリ全体から収集。 +- **探索モード**: `GrammarDiscovery.TypeHierarchy` で名前空間走査を無効化。`Namespace` で従来境界も選択可能。 +- **明示参加**: `[GrammarPart(typeof(Root))]` で名前空間・型階層外の `AstNode` を文法へ追加。 +- **複数文法・方言**: 同じノードを共有する複数文法と、同一ルートの複数 `Mode` を重複生成なしでサポート。 +- **アセンブリ共通Skip**: 宣言済みだが未収集だった `[assembly: Skip(...)]` をGeneratorへ反映。 +- **安定した Core モデル**: `Grammar` が入力コレクションのスナップショットを保持し、`Production` が右辺入力を防御コピー。`GrammarBuilder.Build` を反復可能に。 +- **品質ゲート**: CI と公開フローで全テストを実行。Generator の Core 型競合警告を解消。 + +--- + ## 0.4.0 — 軽量 GLR (LightGlr) モード + 読み取り専用 OnReduce ### ⚠ 破壊的変更 (後方互換性なし) diff --git a/Directory.Build.props b/Directory.Build.props index b0c76e8..33a2c3b 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,6 +1,6 @@ - 0.4.0 + 0.4.1 actbit https://github.com/actbit/AstFirst https://github.com/actbit/AstFirst diff --git a/README.ja.md b/README.ja.md index 3181e45..8ff3e65 100644 --- a/README.ja.md +++ b/README.ja.md @@ -15,13 +15,13 @@ AstFirst はパーサジェネレータ・パーサコンビネータと同じ |---|---|---|---|---| | 文法の定義 | C# のクラス + 属性 | 外部 `.g4` DSL | C# パーサコンビネータ | 該当なし (C#/VB 構文のみ) | | コード生成 | **コンパイル時** (Source Generator) | ビルド時コード生成ツール | **しない** (実行時に解釈) | 該当なし | -| 実行時パースコスト | **ゼロ** — 静的テーブル・ディスパッチなし | 生成コード | 解釈実行 (パース毎にアロケーション/ディスパッチ) | 該当なし | +| 実行時方式 | 静的テーブル、パーサ構築不要 | 生成コード | 解釈実行 (パース毎にアロケーション/ディスパッチ) | 該当なし | | AOT / Native AOT | ✓ 実行時コード生成なし | △ | ✓ | 該当なし | | アルゴリズム | LALR(1) + 軽量 GLR (LightGlr) | LL(\*) / ALL(\*) | 再帰下降コンビネータ | 該当なし | | エラー回復 | Corchuelo et al. ER1/ER2/ER3 (組み込み) | あり | 自作が必要 | 該当なし | | 文法の表現力 | LALR(1) + GLR — `[Precedence]` / fork で衝突解決 | LL(\*) | **チューリング完全** (任意の C# で分岐) | 該当なし | -**コンビネータ (Superpower/Pidgin) に対する強み**: パーサを*コンパイル時*に静的テーブルとして生成するため、解釈もディスパッチもパーサ構築も実行時に走らない。起動・パース毎のコストが事実上ゼロで、AOT/Native AOT にも綺麗に通る。Corchuelo et al. の高品質エラー回復 (ER1 挿入 / ER2 削除 / ER3 Forward move) を組み込み。文法は宣言的な C# なので、IDE のナビゲーションやリファクタリングが効く。 +**コンビネータ (Superpower/Pidgin) に対する強み**: パーサを*コンパイル時*に静的テーブルとして生成するため、実行時の解釈・delegate dispatch・パーサ構築が不要で、AOT/Native AOT にも綺麗に通る。Corchuelo et al. の高品質エラー回復 (ER1 挿入 / ER2 削除 / ER3 Forward move) を組み込み。文法は宣言的な C# なので、IDE のナビゲーションやリファクタリングが効く。 **トレードオフ**: LALR(1) は `[Precedence]`/結合性で shift-reduce 衝突を解決する必要がある。ただし LightGlr モード (`[Grammar(ParseMode = ParseMode.LightGlr)]`) で本質的曖昧性 (cast/paren・generic 等) を並行 fork で扱える。C# 専用のツールチェイン。 @@ -97,14 +97,14 @@ var result = ExprParser.Parse("1+2*3"); // result.HasErrors → false var result2 = ExprParser.Parse("1+"); -// result2.HasErrors → true (panic mode で回復) +// result2.HasErrors → true (Corchuelo ER1/ER2/ER3 で回復) ``` ## 意味解析 AstFirst は構文解析（AST 構築）に加え、意味解析のための標準ヘルパーと 2パスの枠組みを提供する。詳細は [docs/ja/semantic-analysis.md](docs/ja/semantic-analysis.md)。 -- **属性ベースのルール `[OnReduce]` / `[Enter]` / `[Exit]` (推奨)**: 意味ルールを `[Grammar]` ルートクラスの `static` メソッドで書く。Generator がコンストラクタ / Walker から dispatch し、ctx のキャストも自動で挿入（ノード毎のボイラープレート不要）。 +- **属性ベースのルール `[OnReduce]` / `[Enter]` / `[Exit]` (推奨)**: 意味ルールを `[Grammar]` ルートクラスの `static` メソッドで書く。Generator が `[OnReduce]` を Parser の reduce 処理から、`[Enter]`/`[Exit]` を Walker から dispatch し、ctx のキャストも自動で挿入（ノード毎のボイラープレート不要）。 - **1パス目 `OnReduce` (ボトムアップ)**: reduce 時に呼ばれる partial メソッド。`Accept()`/`Reject()` でこの構文を受け入れるか判定（既定 Accept）。`Reject` すると別候補へフォールバック。 - **2パス目 `[Enter]`/`[Exit]` / `OnSecondPassEnter`/`Exit` (トップダウン)**: 生成された汎用 Walker (`{Root}Walker`) が `Parse` 後に `Enter → 子 → Exit` を駆動。スコープ Push/Pop 等の正確な意味解析が書ける。意味フックのない文法では走査を省略（オーバーヘッドなし・ゼロコスト）。 - **型システム**: `TypeSymbol` は継承可能で `FunctionTypeSymbol`/`ArrayTypeSymbol`（共変・反変・構造等価）を組み込み、暗黙の型変換の分類と `OverloadResolver` も提供。`BasicSemanticContext` は `TypeContext` を標準で保持。 @@ -227,11 +227,12 @@ var result = ProgramParser.Parse(code, new MiniCContext()); | 属性 | 対象 | 役割 | |---|---|---| | `[Grammar]` | クラス | 文法の開始記号（ルート非終端）。Generator の抽出開始点。`Mode` で複数方言を切り替え。 | +| `[GrammarPart(typeof(Root))]` | クラス | 名前空間・ルート型階層外のノードを文法へ明示的に追加。`[Grammar].Discovery` で探索方法を選択可能。 | | `[Rule]` | static メソッド | 生成規則（1クラス1つ）。メソッドの**引数**が右辺。 | | `[Token(@"regex")]` / `[Pattern(@"regex")]` | `[Rule]` メソッドの `Token` 引数 | 字句ルール（正規表現）。`Priority` でレクサ優先度（大きいほど高優先）。 | | `[Precedence(n)]` | クラス（演算ノード） | 演算子優先度/結合性。`n` が大きいほど高優先。`IsRightAssociative`/`IsNonAssociative` で結合性。 | | `[Repeat]` / `[Repeat(Min=0)]` | `[Rule]` メソッドの `AstNode` 派生引数 | リスト（繰り返し）。`Min=1`（既定）は1回以上、`Min=0` は0回以上（空リスト可）。`IReadOnlyList` に展開。 | -| `[Skip(@"regex")]` | クラス（`[Grammar]` と同じ） | スキップパターン（空白・コメント等）。 | +| `[Skip(@"regex")]` | `[Grammar]` クラス／アセンブリ | スキップパターン（空白・コメント等）。アセンブリ指定は全Grammar共通。 | ### `[Rule]` メソッドの引数（型ベース分類） @@ -365,7 +366,7 @@ AstFirst.slnx ## テスト -352 テスト（AstFirst.Tests 299 + Generator.Tests 53）。レクサ/DFA/LALR の各段階、エンドツーエンド、エラー回復 (Corchuelo)、GLR fork/dedup、意味解析（スコープ・2パス目・型チェック・ctx → `ParseResult.Diagnostics` の統合）、`Accept`/`Reject` フォールバック、`OnAccepted` コールバック、位置情報（行・列）を検証。 +374 テスト（AstFirst.Tests 308 + Generator.Tests 66）。レクサ/DFA/LALR の各段階、エンドツーエンド、エラー回復 (Corchuelo)、GLR fork/dedup、意味解析、文法ノード探索、Coreビルドのスナップショット安定性、`Accept`/`Reject` フォールバック、`OnAccepted`、位置情報を検証。 ## ライセンス diff --git a/README.md b/README.md index a0bff73..41bd998 100644 --- a/README.md +++ b/README.md @@ -15,13 +15,13 @@ AstFirst sits in the same space as parser generators and combinator libraries, b |---|---|---|---|---| | Grammar in | plain C# classes + attributes | external `.g4` DSL | C# parser combinators | n/a — C#/VB syntax only | | Code generated | **compile time** (Source Generator) | build-time codegen tool | **never** — interpreted at runtime | n/a | -| Runtime parse cost | **zero** — static tables, no dispatch | generated code | interpreted (allocation/dispatch per parse) | n/a | +| Runtime strategy | static tables, no parser construction | generated code | interpreted (allocation/dispatch per parse) | n/a | | AOT / Native AOT | ✓ no runtime codegen | △ | ✓ | n/a | | Algorithm | LALR(1) + Lightweight GLR (LightGlr) | LL(\*) / ALL(\*) | recursive-descent combinators | n/a | | Error recovery | Corchuelo et al. ER1/ER2/ER3 (built-in) | yes | hand-rolled | n/a | | Grammar power | LALR(1) + GLR — resolve conflicts with `[Precedence]` / fork | LL(\*) | **Turing-complete** (branch on any C#) | n/a | -**Strengths vs combinators (Superpower/Pidgin)**: the parser is emitted *at compile time* as static tables — no interpretation, no delegate dispatch, no per-parse construction. Startup and per-parse cost are effectively zero, it AOTs / Native-AOTs cleanly, and Corchuelo et al. error recovery (ER1 insert / ER2 delete / ER3 Forward move) is built in. The grammar is declarative C#, so the IDE can navigate and refactor it. +**Strengths vs combinators (Superpower/Pidgin)**: the parser is emitted *at compile time* as static tables, so runtime interpretation, delegate dispatch, and parser construction are unnecessary. It AOTs / Native-AOTs cleanly, and Corchuelo et al. error recovery (ER1 insert / ER2 delete / ER3 Forward move) is built in. The grammar is declarative C#, so the IDE can navigate and refactor it. **Trade-offs**: LALR(1) needs `[Precedence]`/associativity to resolve shift-reduce conflicts. However, LightGlr mode (`[Grammar(ParseMode = ParseMode.LightGlr)]`) handles inherent ambiguity (cast/paren, generics) via parallel fork. A C#-only toolchain. @@ -97,14 +97,14 @@ var result = ExprParser.Parse("1+2*3"); // result.HasErrors -> false var result2 = ExprParser.Parse("1+"); -// result2.HasErrors -> true (recovered via panic mode) +// result2.HasErrors -> true (recovered via Corchuelo ER1/ER2/ER3) ``` ## Semantic analysis AstFirst provides standard helpers and a two-pass framework for semantic analysis on top of parsing. See [docs/en/semantic-analysis.md](docs/en/semantic-analysis.md) for details. -- **Attribute-based rules `[OnReduce]` / `[Enter]` / `[Exit]` (recommended)**: write semantic rules as `static` methods on the `[Grammar]` root class. The generator dispatches them from the constructor / Walker and injects the `ctx` cast for you — no per-node boilerplate. +- **Attribute-based rules `[OnReduce]` / `[Enter]` / `[Exit]` (recommended)**: write semantic rules as `static` methods on the `[Grammar]` root class. The generator dispatches `[OnReduce]` during parser reduction and `[Enter]`/`[Exit]` from the Walker, injecting the `ctx` cast for you — no per-node boilerplate. - **First pass `OnReduce` (bottom-up)**: a partial method called at reduce time. `Accept()`/`Reject()` decides whether to accept this interpretation (default Accept). `Reject` falls back to the next candidate. - **Second pass `[Enter]`/`[Exit]` / `OnSecondPassEnter`/`Exit` (top-down)**: a generated generic Walker (`{Root}Walker`) drives `Enter -> children -> Exit` after `Parse`. Accurate semantic analysis like scope Push/Pop fits here. Grammars with no semantic hook skip the traversal entirely (no overhead, zero-cost). - **Type system**: `TypeSymbol` is inheritable with built-in `FunctionTypeSymbol`/`ArrayTypeSymbol` (variance + structural equality), plus implicit-conversion classification and `OverloadResolver`. `BasicSemanticContext` carries a `TypeContext` by default. @@ -227,11 +227,12 @@ See [docs/en/grammar-reference.md](docs/en/grammar-reference.md) for details. | Attribute | Target | Role | |---|---|---| | `[Grammar]` | class | Start symbol (root nonterminal). Generator's extraction entry point. `Mode` switches dialects. | +| `[GrammarPart(typeof(Root))]` | class | Explicitly adds a node outside the grammar namespace/root hierarchy. `[Grammar].Discovery` selects discovery behavior. | | `[Rule]` | static method | A production (one per class). The method's **parameters** are the RHS. | | `[Token(@"regex")]` / `[Pattern(@"regex")]` | `Token` parameter of a `[Rule]` method | Lexical rule (regex). `Priority` sets lexer priority (higher wins). | | `[Precedence(n)]` | class (operator node) | Operator precedence/associativity. Higher `n` binds tighter. `IsRightAssociative`/`IsNonAssociative`. | | `[Repeat]` / `[Repeat(Min=0)]` | `AstNode`-derived parameter of a `[Rule]` method | List (repetition). `Min=1` (default) = one or more, `Min=0` = zero or more (empty list allowed). Expands to `IReadOnlyList`. | -| `[Skip(@"regex")]` | class (same as `[Grammar]`) | Skip pattern (whitespace, comments). | +| `[Skip(@"regex")]` | `[Grammar]` class / assembly | Skip pattern (whitespace, comments). Assembly-level patterns apply to every grammar. | ### `[Rule]` method parameters (type-based classification) @@ -316,7 +317,7 @@ Japanese versions are under `docs/ja/` and [README.md](README.md). ## Tests -352 tests (AstFirst.Tests 299 + Generator.Tests 53). Covers lexer/DFA/LALR stages, end-to-end, error recovery (Corchuelo), GLR fork/dedup, semantic analysis (scopes, two-pass, type checking, ctx -> `ParseResult.Diagnostics` integration), `Accept`/`Reject` fallback, `OnAccepted` callback, and positions (line/column). +374 tests (AstFirst.Tests 308 + Generator.Tests 66). Covers lexer/DFA/LALR stages, end-to-end, error recovery (Corchuelo), GLR fork/dedup, semantic analysis, grammar discovery, Core build snapshot stability, `Accept`/`Reject` fallback, `OnAccepted`, and positions. ## License diff --git a/docs/en/architecture.md b/docs/en/architecture.md index a015df9..4e6c32c 100644 --- a/docs/en/architecture.md +++ b/docs/en/architecture.md @@ -13,7 +13,7 @@ AstFirst is a parser generator built as three layers plus a generator. ## Generator pipeline -1. **Extraction** (`ModelExtraction`): traverses AstNode/Token derivatives and `[Pattern]` from the `[Grammar]` root, collects `[OnReduce]`/`[Enter]`/`[Exit]` attribute semantic rules, and converts them into an equality-comparable POCO model (`GrammarModel`/`AnalyzeRuleModel`). +1. **Extraction** (`ModelExtraction`): follows `[Grammar].Discovery` to collect nodes from the root namespace, root type hierarchy, and `[GrammarPart]`; then converts `[Rule]`/`[Token]` and semantic rules into equality-comparable POCO models. 2. **DFA build** (`ModelToDfa`): regex of each rule -> NFA (Thompson) -> DFA (subset construction) -> minimization (Hopcroft). 3. **LALR table** (`ModelToTable`): LR(0) automaton -> FIRST/NULLABLE -> DeRemer-Pennello lookahead propagation -> ACTION/GOTO tables + conflict detection. 4. **Code emission** (`CodeEmitter` / `ParserEmitter` / `WalkerEmitter`): generates C# for Lexer (DFA arrays), Parser (LALR table + shift/reduce driver), Walker (Enter/Exit/Walk + `[Enter]`/`[Exit]` dispatch), and per-node partials (including `[OnReduce]` dispatch). @@ -21,7 +21,7 @@ AstFirst is a parser generator built as three layers plus a generator. ## Generated code shape - **Lexer**: embeds the DFA transition table and accepting rules in `static readonly` arrays; `Tokenize()` runs longest-match + priority-driven. Also computes each token's 1-based line/column. -- **Parser**: embeds ACTION/GOTO tables and Productions in arrays and drives shift/reduce/accept. At reduce it calls the AST class constructor to build the AST (`[OnReduce]` attribute methods are called right after partial `OnReduce`). Includes panic-mode error recovery. +- **Parser**: embeds ACTION/GOTO tables and Productions in arrays and drives shift/reduce/accept. At reduce it calls the AST class constructor to build the AST (`[OnReduce]` attribute methods are called right after partial `OnReduce`). Includes Corchuelo ER1/ER2/ER3 error repair. - **Walker**: `EnterXxx` / `ExitXxx` (virtual, empty) per concrete node + `Walk` (iterative stack: Enter -> children -> Exit). Also invokes `IOnSecondPassEnter`/`Exit` and `[Enter]`/`[Exit]` attribute methods. Children are collected from each node's public properties of AstNode-derived types. If a grammar has no semantic hook at all, the Walker is not emitted (zero-cost). ## Caching strategy diff --git a/docs/en/grammar-reference.md b/docs/en/grammar-reference.md index d238260..3847d79 100644 --- a/docs/en/grammar-reference.md +++ b/docs/en/grammar-reference.md @@ -9,12 +9,13 @@ AstFirst grammars are written with C# classes and attributes. The generator emit | Attribute | Target | Role | |---|---|---| | `[Grammar]` | class | Start symbol (root nonterminal). Generator's extraction entry point. `Mode` switches dialects. | +| `[GrammarPart(typeof(Root))]` | class | Explicitly includes an `AstNode` outside the grammar namespace/root hierarchy. | | `[Rule]` | static method | A production. The method's **parameters** are the RHS. Multiple per class allowed (see below). | | `[Token(@"regex")]` / `[Pattern(@"regex")]` | `Token` parameter of a `[Rule]` method | Lexical rule (regex). `Priority` sets lexer priority, `Kind` sets token category. | | `[Precedence(n)]` | class (operator node) | Operator precedence/associativity. Higher `n` binds tighter. | | `[Repeat]` / `[Repeat(Min=0)]` | `AstNode`-derived parameter of a `[Rule]` method | List (repetition). `Min=1` (default) = one or more, `Min=0` = zero or more. Expands to `IReadOnlyList`. | -| `[Skip(@"regex")]` | class (same as `[Grammar]`) | Skip pattern (whitespace, comments). | -| `[OnReduce]` / `[Enter]` / `[Exit]` | static method (on the `[Grammar]` root class) | Semantic rule. The generator dispatches it from the constructor (`[OnReduce]`) / Walker (`[Enter]`/`[Exit]`); the ctx cast is injected. | +| `[Skip(@"regex")]` | `[Grammar]` class / assembly | Skip pattern (whitespace, comments). Assembly-level patterns apply to every grammar. | +| `[OnReduce]` / `[Enter]` / `[Exit]` | static method (on the `[Grammar]` root class) | Semantic rule. The generator dispatches `[OnReduce]` during parser reduction and `[Enter]`/`[Exit]` from the Walker; the ctx cast is injected. | ## `[Grammar]` @@ -28,6 +29,28 @@ public abstract partial class Expr : AstNode { } The `Mode` named property switches dialects (see below). +### GrammarDiscovery (node discovery) + +Use `Discovery` to select how grammar nodes are found. + +| Value | Behavior | +|---|---| +| `NamespaceAndTypeHierarchy` (default) | Includes `AstNode` types in the root namespace, root-derived types anywhere in the assembly, and explicit `[GrammarPart]` types. | +| `TypeHierarchy` | Disables namespace scanning; includes root-derived types anywhere in the assembly and explicit `[GrammarPart]` types. | +| `Namespace` | Legacy boundary; includes `AstNode` types in the root namespace and explicit `[GrammarPart]` types. | + +```csharp +[Grammar(Discovery = GrammarDiscovery.TypeHierarchy)] +public abstract partial class Expr : AstNode { } + +// Discovered across namespaces because it derives from Expr. +public sealed partial class NumberExpr : Expr { } + +// A shared node outside the hierarchy can opt in explicitly. +[GrammarPart(typeof(Expr))] +public sealed partial class SharedValue : AstNode { } +``` + ### ParseMode (parser execution mode) The `ParseMode` named property selects the parser execution mode. Default is `Lalr` (deterministic LALR(1)). @@ -48,7 +71,7 @@ The `ParseMode` named property selects the parser execution mode. Default is `La - **N=3 and costs are fixed**: Forward move symbols `N=3`, insert cost=1/delete cost=2 are hardcoded. The Corchuelo paper recommends per-language tuning; not yet supported. - **Single-pass repair (no recursion)**: The original Corchuelo applies ER1/ER2/ER3 recursively; this implementation applies one round only. Consecutive errors are repaired one at a time on subsequent dead states. - **SimulateForward checks the first path only**: Does not fork at conflict cells during simulation, so full agreement with production fork paths is not guaranteed. -- **Error recovery behavior**: LightGlr's Corchuelo repair differs from the panic-mode recovery used in Lalr mode — it inserts/deletes tokens to continue parsing. The same input may produce different error positions/messages depending on the mode. +- **Error recovery behavior**: Both Lalr and LightGlr use Corchuelo ER1/ER2/ER3 to insert/delete tokens and continue parsing. GLR branching can still produce different error positions/messages between modes for the same input. ### ⚠ Breaking Changes (0.4.0) @@ -197,10 +220,11 @@ public sealed partial class NonEmpty : Program ## `[Skip]` -Skip pattern (whitespace, comments). Attach to the same class as `[Grammar]`. Matched spans are removed from the token stream. +Skip pattern (whitespace, comments). Attach it to a `[Grammar]` class for one grammar or to the assembly for every grammar. Matched spans are removed from the token stream. ```csharp [Skip(@"(\s|//[^\n]*)+")] // whitespace and line comments +[assembly: Skip(@"//[^\n]*")] // shared by every grammar ``` ## Writing grammar @@ -242,7 +266,7 @@ Special parameter types of a `[Rule]` method: - **`OnReduce(ctx)`**: a partial method called when a rule is reduced (bottom-up). Child properties and `Span` (auto-computed from children) are already set. Use `this.RuleName` to branch on the rule, override `Span`, etc. - **Accept/Reject**: override `IsAccepted` to return `false` to reject a reduce and try fallback candidates. See the [README](../../README.md) "Accept/Reject and fallback" section. - **`OnSecondPass`**: the second-pass traversal (top-down). For nodes implementing `IOnSecondPassEnter`/`IOnSecondPassExit`, the generator calls `OnSecondPassEnter` (before children) → recurse children → `OnSecondPassExit` (after children). -- **`[OnReduce]` / `[Enter]` / `[Exit]` attributes**: attach to a `static` method on the `[Grammar]` root class and the generator dispatches it from the Walker / constructor (the ctx cast is injected automatically). `[OnReduce]` runs at reduce; `[Enter]`/`[Exit]` run in the second pass. See the [semantic analysis guide](semantic-analysis.md). +- **`[OnReduce]` / `[Enter]` / `[Exit]` attributes**: attach to a `static` method on the `[Grammar]` root class. The generator dispatches `[OnReduce]` during parser reduction and `[Enter]`/`[Exit]` from the Walker (the ctx cast is injected automatically). `[OnReduce]` runs at reduce; `[Enter]`/`[Exit]` run in the second pass. See the [semantic analysis guide](semantic-analysis.md). ## Dialects (Mode) diff --git a/docs/ja/architecture.md b/docs/ja/architecture.md index 3d308ab..9ae25df 100644 --- a/docs/ja/architecture.md +++ b/docs/ja/architecture.md @@ -13,7 +13,7 @@ AstFirst は 3 層 + Generator で構成されるパーサジェネレータ。 ## Generator の処理フロー -1. **抽出** (`ModelExtraction`): `[Grammar]` ルートから AstNode 派生・Token 派生・`[Pattern]` を走査し、`[OnReduce]`/`[Enter]`/`[Exit]` 属性付き意味規則を収集して、等価比較可能な POCO モデル (`GrammarModel`/`AnalyzeRuleModel`) に変換。 +1. **抽出** (`ModelExtraction`): `[Grammar]` の `Discovery` に従い、同一名前空間・ルート型階層・`[GrammarPart]` から文法ノードを収集する。`[Rule]`/`[Token]` と意味規則を等価比較可能な POCO モデルへ変換する。 2. **DFA 構築** (`ModelToDfa`): 字句ルールの正規表現 → NFA (Thompson) → DFA (部分集合構成法) → 最小化 (Hopcroft)。 3. **LALR テーブル** (`ModelToTable`): LR(0) オートマトン → FIRST/NULLABLE → DeRemer-Pennello ルックアヘッド伝播 → ACTION/GOTO テーブル + 衝突検出。 4. **コード生成** (`CodeEmitter` / `ParserEmitter` / `WalkerEmitter`): Lexer（DFA 配列）、Parser（LALR テーブル + shift/reduce 駆動）、Walker（Enter/Exit/Walk + `[Enter]`/`[Exit]` dispatch）、各ノードの partial（`[OnReduce]` dispatch 含む）の C# コードを生成。 @@ -21,7 +21,7 @@ AstFirst は 3 層 + Generator で構成されるパーサジェネレータ。 ## 生成コードの構造 - **Lexer**: DFA の遷移表と受理ルールを `static readonly` 配列に埋め込み、`Tokenize()` で最長一致 + 優先度駆動。各トークンの行・列（1 ベース）も計算。 -- **Parser**: ACTION/GOTO テーブル + Productions を配列に埋め込み、shift/reduce/accept を駆動。reduce 時に AST クラスのコンストラクタを呼び AST を構築（`[OnReduce]` 属性メソッドは partial `OnReduce` の直後に呼出）。panic mode のエラー回復付き。 +- **Parser**: ACTION/GOTO テーブル + Productions を配列に埋め込み、shift/reduce/accept を駆動。reduce 時に AST クラスのコンストラクタを呼び AST を構築（`[OnReduce]` 属性メソッドは partial `OnReduce` の直後に呼出）。Corchuelo ER1/ER2/ER3 エラー修復付き。 - **Walker**: 各具象ノードの `EnterXxx` / `ExitXxx`（virtual、空実装）+ `Walk`（反復スタックで Enter → 子 → Exit）。`IOnSecondPassEnter`/`Exit` と `[Enter]`/`[Exit]` 属性メソッドも呼出。子は各ノードの public プロパティから AstNode 派生を収集して辿る。意味解析フックが1つもない文法では生成を省略（ゼロコスト）。 ## キャッシュ戦略 diff --git a/docs/ja/grammar-reference.md b/docs/ja/grammar-reference.md index 6380a2d..bc33160 100644 --- a/docs/ja/grammar-reference.md +++ b/docs/ja/grammar-reference.md @@ -9,12 +9,13 @@ AstFirst では C# のクラスと属性で文法を書く。Generator がコン | 属性 | 対象 | 役割 | |---|---|---| | `[Grammar]` | クラス | 文法の開始記号（ルート非終端）。Generator の抽出開始点。`Mode` で複数方言を切り替え。 | +| `[GrammarPart(typeof(Root))]` | クラス | 名前空間・ルート型階層外の `AstNode` を指定文法へ明示的に参加させる。 | | `[Rule]` | static メソッド | 生成規則。メソッドの**引数**が右辺。1クラスに複数置ける（後述）。 | | `[Token(@"regex")]` / `[Pattern(@"regex")]` | `[Rule]` メソッドの `Token` 引数 | 字句ルール（正規表現）。`Priority` でレクサ優先度、`Kind` でトークン種別。 | | `[Precedence(n)]` | クラス（演算ノード） | 演算子優先度/結合性。`n` が大きいほど高優先。 | | `[Repeat]` / `[Repeat(Min=0)]` | `[Rule]` メソッドの `AstNode` 派生引数 | リスト（繰り返し）。`Min=1`（既定）= 1回以上、`Min=0` = 0回以上。`IReadOnlyList` に展開。 | -| `[Skip(@"regex")]` | クラス（`[Grammar]` と同じ） | スキップパターン（空白・コメント等）。 | -| `[OnReduce]` / `[Enter]` / `[Exit]` | static メソッド（`[Grammar]` ルートクラス） | 意味ルール。Generator がコンストラクタ（`[OnReduce]`）/ Walker（`[Enter]`/`[Exit]`）から dispatch（ctx キャスト自動注入）。 | +| `[Skip(@"regex")]` | `[Grammar]` クラス／アセンブリ | スキップパターン（空白・コメント等）。アセンブリ指定は全Grammar共通。 | +| `[OnReduce]` / `[Enter]` / `[Exit]` | static メソッド（`[Grammar]` ルートクラス） | 意味ルール。Generator が `[OnReduce]` を Parser の reduce 処理から、`[Enter]`/`[Exit]` を Walker から dispatch（ctx キャスト自動注入）。 | ## `[Grammar]` @@ -28,6 +29,28 @@ public abstract partial class Expr : AstNode { } `Mode` 名前付きプロパティで複数方言を切り替えられる（後述）。 +### GrammarDiscovery（ノード探索） + +`Discovery` で文法ノードの探索範囲を選択する。 + +| 値 | 動作 | +|---|---| +| `NamespaceAndTypeHierarchy`（既定） | ルートと同じ名前空間の `AstNode` 派生型、アセンブリ内のルート派生型、明示的な `[GrammarPart]` を収集する。 | +| `TypeHierarchy` | 名前空間を走査せず、アセンブリ内のルート派生型と明示的な `[GrammarPart]` だけを収集する。 | +| `Namespace` | 従来互換。同じ名前空間の `AstNode` 派生型と明示的な `[GrammarPart]` を収集する。 | + +```csharp +[Grammar(Discovery = GrammarDiscovery.TypeHierarchy)] +public abstract partial class Expr : AstNode { } + +// 別名前空間でも Expr 派生型なので自動収集される。 +public sealed partial class NumberExpr : Expr { } + +// Expr 派生でない共有ノードは明示的に参加させられる。 +[GrammarPart(typeof(Expr))] +public sealed partial class SharedValue : AstNode { } +``` + ### ParseMode（パーサの実行モード） `ParseMode` 名前付きプロパティでパーサの実行モードを切り替えられる。既定は `Lalr`（確定 LALR(1)）。 @@ -48,7 +71,7 @@ public abstract partial class Expr : AstNode { } - **N=3・コスト固定**: ER3 の Forward move 確認シンボル数 `N=3`、挿入コスト=1/削除コスト=2 は固定値。Corchuelo 論文では言語に応じたチューニングを推奨しているが、本実装では未対応。 - **1 回修復 (再帰なし)**: Corchuelo 本来は ER1/ER2/ER3 を再帰的に適用するが、本実装は1回の ER1/ER2 + ER3 のみ。連続エラーは次の dead で順次修復される。 - **SimulateForward は最初の経路のみ確認**: コンフリクトセルでも fork せず最初の shift/reduce のみ追うため、本番の fork 経路との完全一致は保証しない。 -- **エラー回復の挙動**: LightGlr の Corchuelo 修復は、LALR モードの panic mode 回復とは異なり、トークンを補完/削除してパースを続行する。同じ入力でもモードによりエラー位置・メッセージが変わる場合がある。 +- **エラー回復の挙動**: LALR・LightGlr とも Corchuelo ER1/ER2/ER3 でトークンを補完/削除してパースを続行する。GLRの分岐により、同じ入力でもモードごとにエラー位置・メッセージが変わる場合がある。 ### ⚠ バージョン互換性のない変更 (0.4.0) @@ -197,10 +220,11 @@ public sealed partial class NonEmpty : Program ## `[Skip]` -スキップパターン（空白・コメント等）。`[Grammar]` を付けたクラスに併せて付ける。マッチした部分はトークン列から除外される。 +スキップパターン（空白・コメント等）。`[Grammar]` クラスに付けるとその文法へ、アセンブリに付けると全Grammarへ適用される。マッチした部分はトークン列から除外される。 ```csharp [Skip(@"(\s|//[^\n]*)+")] // 空白と行コメント +[assembly: Skip(@"//[^\n]*")] // 全Grammar共通 ``` ## 規則の書き方 @@ -242,7 +266,7 @@ public sealed partial class AAdd : ABinary - **`OnReduce(ctx)`**: 規則が reduce されたとき（ボトムアップ）に呼ばれる partial メソッド。子プロパティと `Span`（子から自動計算）が既に設定済み。`this.RuleName` で規則を判定、`Span` を上書きする等を行う。 - **Accept/Reject**: `IsAccepted` をオーバーライドして `false` を返すと、その reduce を拒否（Reject）しフォールバック候補を試す。詳細は [README](../../README.md) の「Accept/Reject とフォールバック」。 - **`OnSecondPass`**: 2パス目のトラバーサル（トップダウン）。`IOnSecondPassEnter`/`IOnSecondPassExit` を実装したノードに対し、`OnSecondPassEnter`（子の前）→ 子再帰 → `OnSecondPassExit`（子の後）を自動呼出。 -- **`[OnReduce]` / `[Enter]` / `[Exit]` 属性**: `[Grammar]` ルートクラスの `static` メソッドに付けると、Generator が Walker / コンストラクタに dispatch する（ctx キャストは自動注入）。`[OnReduce]` は reduce 時、`[Enter]`/`[Exit]` は 2パス目。詳細は [意味解析ガイド](semantic-analysis.md)。 +- **`[OnReduce]` / `[Enter]` / `[Exit]` 属性**: `[Grammar]` ルートクラスの `static` メソッドに付けると、Generator が `[OnReduce]` を Parser の reduce 処理から、`[Enter]`/`[Exit]` を Walker から dispatch する（ctx キャストは自動注入）。`[OnReduce]` は reduce 時、`[Enter]`/`[Exit]` は 2パス目。詳細は [意味解析ガイド](semantic-analysis.md)。 ## 複数方言（Mode） diff --git a/src/AstFirst.Core/Parsing/Grammar.cs b/src/AstFirst.Core/Parsing/Grammar.cs index d8786cf..ce121d3 100644 --- a/src/AstFirst.Core/Parsing/Grammar.cs +++ b/src/AstFirst.Core/Parsing/Grammar.cs @@ -32,15 +32,17 @@ public Grammar( IReadOnlyList? unreachableNonTerminals = null, IReadOnlyList? undefinedNonTerminals = null) { - Productions = productions; - Symbols = symbols; + Productions = productions.ToArray(); + Symbols = symbols.ToArray(); StartSymbol = startSymbol; AugmentedStart = augmentedStart; EndOfFile = endOfFile; AugmentedProduction = augmentedProduction; - TerminalPrecedence = terminalPrecedence ?? new Dictionary(); - UnreachableNonTerminals = unreachableNonTerminals ?? Array.Empty(); - UndefinedNonTerminals = undefinedNonTerminals ?? Array.Empty(); + TerminalPrecedence = terminalPrecedence is null + ? new Dictionary() + : terminalPrecedence.ToDictionary(pair => pair.Key, pair => pair.Value); + UnreachableNonTerminals = unreachableNonTerminals?.ToArray() ?? Array.Empty(); + UndefinedNonTerminals = undefinedNonTerminals?.ToArray() ?? Array.Empty(); } } @@ -98,7 +100,6 @@ public Grammar Build(Symbol startSymbol) var augStart = GetOrAdd(startSymbol.Name + "'", isTerminal: false); var eof = GetOrAdd("$", isTerminal: true); var augProd = new Core.Parsing.Production(_productions.Count, augStart, new[] { startSymbol, eof }); - _productions.Add(augProd); // 到達不能/未定義非終端を検出。 var reachable = ComputeReachable(startSymbol); @@ -112,7 +113,8 @@ public Grammar Build(Symbol startSymbol) if (!s.IsTerminal) rightSideNonTerminals.Add(s); var undefined = rightSideNonTerminals.Where(nt => !lhsNonTerminals.Contains(nt)).ToList(); - return new Grammar(_productions, _symbolList, startSymbol, augStart, eof, augProd, _terminalPrecedence, unreachable, undefined); + var productions = new List(_productions) { augProd }; + return new Grammar(productions, _symbolList, startSymbol, augStart, eof, augProd, _terminalPrecedence, unreachable, undefined); } /// 開始記号から到達可能な非終端を BFS で集める。 diff --git a/src/AstFirst.Core/Parsing/Production.cs b/src/AstFirst.Core/Parsing/Production.cs index c5d41a5..d127d64 100644 --- a/src/AstFirst.Core/Parsing/Production.cs +++ b/src/AstFirst.Core/Parsing/Production.cs @@ -1,3 +1,4 @@ +using System; namespace AstFirst.Core.Parsing; /// 生成規則 LHS -> Rhs[0] Rhs[1] ...。Tag には AST クラス等のメタ情報を載せる。 @@ -16,7 +17,8 @@ public Production(int id, Symbol lhs, Symbol[] rhs, object? tag = null, Preceden { Id = id; Lhs = lhs; - Rhs = rhs; + if (rhs is null) throw new ArgumentNullException(nameof(rhs)); + Rhs = (Symbol[])rhs.Clone(); Tag = tag; RulePrecedence = rulePrecedence; } diff --git a/src/AstFirst.Generator/AstFirst.Generator.csproj b/src/AstFirst.Generator/AstFirst.Generator.csproj index b9eff73..c57007f 100644 --- a/src/AstFirst.Generator/AstFirst.Generator.csproj +++ b/src/AstFirst.Generator/AstFirst.Generator.csproj @@ -50,4 +50,11 @@ + + + + + + + diff --git a/src/AstFirst.Generator/CodeEmitter.cs b/src/AstFirst.Generator/CodeEmitter.cs index 30edbb2..de103a7 100644 --- a/src/AstFirst.Generator/CodeEmitter.cs +++ b/src/AstFirst.Generator/CodeEmitter.cs @@ -89,7 +89,7 @@ public static (string ns, string type) SplitFullName(string fullName) /// [Rule] 引数名 (lowerCamel) をプロパティ名 (PascalCase) に。空なら "Item"。 public static string Pascalize(string? s) - => string.IsNullOrEmpty(s) ? "Item" : char.ToUpperInvariant(s[0]) + s.Substring(1); + => string.IsNullOrEmpty(s) ? "Item" : char.ToUpperInvariant(s![0]) + s.Substring(1); private static string Escape(string s) => s.Replace("\\", "\\\\").Replace("\"", "\\\""); } diff --git a/src/AstFirst.Generator/GlrParserEmitter.cs b/src/AstFirst.Generator/GlrParserEmitter.cs index e98adab..378aa61 100644 --- a/src/AstFirst.Generator/GlrParserEmitter.cs +++ b/src/AstFirst.Generator/GlrParserEmitter.cs @@ -185,7 +185,7 @@ private static void EmitGlrHelpers(StringBuilder sb, Grammar grammar, GrammarMod { case ReduceActionModel action: { - sb.Append(" case ").Append(prod.Id).Append(": { return new ").Append(action.AstTypeName).Append("(\"").Append(action.RuleName).Append("\""); + sb.Append(" case ").Append(prod.Id).Append(": { var __node = new ").Append(action.AstTypeName).Append("(\"").Append(action.RuleName).Append("\""); for (int j = 0; j < action.Parameters.Count; j++) { sb.Append(", "); @@ -197,7 +197,9 @@ private static void EmitGlrHelpers(StringBuilder sb, Grammar grammar, GrammarMod } else sb.Append("(").Append(p.CastTypeName).Append(")children[").Append(p.ChildIndex).Append("]!"); } - sb.AppendLine("); }"); + sb.Append("); "); + ParserEmitter.EmitOnReduceAnalyzeRules(sb, model, action.AstTypeName, "__node"); + sb.AppendLine("return __node; }"); break; } case ListReduceActionModel listAction: diff --git a/src/AstFirst.Generator/ModelExtraction.cs b/src/AstFirst.Generator/ModelExtraction.cs index 6558a3e..00ce145 100644 --- a/src/AstFirst.Generator/ModelExtraction.cs +++ b/src/AstFirst.Generator/ModelExtraction.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.Collections.Immutable; using System.Linq; using Microsoft.CodeAnalysis; @@ -15,14 +16,24 @@ public static class ModelExtraction private const string TokenFullName = "AstFirst.Token"; private const string SemanticContextFullName = "AstFirst.SemanticContext"; - public static GrammarModel? Extract(GeneratorAttributeSyntaxContext context) + public static ImmutableArray ExtractAll(GeneratorAttributeSyntaxContext context) { - if (context.TargetSymbol is not INamedTypeSymbol rootType) return null; + if (context.TargetSymbol is not INamedTypeSymbol rootType) return ImmutableArray.Empty; var location = context.TargetNode?.GetLocation(); - return Extract(context.SemanticModel.Compilation, rootType, location); + var models = ImmutableArray.CreateBuilder(context.Attributes.Length); + foreach (var grammarAttribute in context.Attributes) + models.Add(Extract(context.SemanticModel.Compilation, rootType, location, grammarAttribute)); + return models.MoveToImmutable(); } public static GrammarModel Extract(Compilation compilation, INamedTypeSymbol rootType, Location? rootLocation = null) + { + var grammarAttribute = rootType.GetAttributes() + .LastOrDefault(attribute => IsAstFirstAttribute(attribute, "GrammarAttribute")); + return Extract(compilation, rootType, rootLocation, grammarAttribute); + } + + private static GrammarModel Extract(Compilation compilation, INamedTypeSymbol rootType, Location? rootLocation, AttributeData? grammarAttribute) { var astNodeBase = compilation.GetTypeByMetadataName(AstNodeFullName); var tokenBase = compilation.GetTypeByMetadataName(TokenFullName); @@ -30,26 +41,39 @@ public static GrammarModel Extract(Compilation compilation, INamedTypeSymbol roo var secondPassEnter = compilation.GetTypeByMetadataName("AstFirst.IOnSecondPassEnter"); var secondPassExit = compilation.GetTypeByMetadataName("AstFirst.IOnSecondPassExit"); + string? mode = null; + var parseMode = ParseMode.Lalr; + var discovery = GrammarDiscovery.NamespaceAndTypeHierarchy; + if (grammarAttribute is not null) + foreach (var na in grammarAttribute.NamedArguments) + { + if (na.Key == "Mode" && na.Value.Value is string m) mode = m; + if (na.Key == "ParseMode" && na.Value.Value is int pm) parseMode = (ParseMode)pm; + if (na.Key == "Discovery" && na.Value.Value is int d) discovery = (GrammarDiscovery)d; + } + var nodes = new List(); var tokenDefs = new List(); var tokenDerivedWarnings = new List(); - foreach (var type in GetAllTypes(compilation.Assembly.GlobalNamespace)) + var allTypes = GetAllTypes(compilation.Assembly.GlobalNamespace).ToList(); + foreach (var type in allTypes) { if (type.TypeKind != Microsoft.CodeAnalysis.TypeKind.Class) continue; if (type.DeclaredAccessibility != Accessibility.Public) continue; - // rootType と同じ名前空間の型のみ収集 (別文法の混入を防ぎ、到達不能/未定義検知の誤検知を避ける)。 - if (!SymbolEqualityComparer.Default.Equals(type.ContainingNamespace, rootType.ContainingNamespace)) continue; - - if (astNodeBase is not null && InheritsFrom(type, astNodeBase)) - nodes.Add(ExtractNode(type, contextBase, astNodeBase, tokenBase, secondPassEnter, secondPassExit)); - if (tokenBase is not null && InheritsFrom(type, tokenBase)) + bool sameNamespace = SymbolEqualityComparer.Default.Equals(type.ContainingNamespace, rootType.ContainingNamespace); + bool inRootHierarchy = InheritsFromOrEquals(type, rootType); + bool explicitPart = IsGrammarPart(type, rootType); + bool includeNode = discovery switch { - // G7: Token派生型に (string) コンストラクタが必要 (new DerivedType(token.Text) の生成)。 - if (!HasStringConstructor(type)) - tokenDerivedWarnings.Add(type.ToDisplayString()); - } + GrammarDiscovery.TypeHierarchy => inRootHierarchy || explicitPart, + GrammarDiscovery.Namespace => sameNamespace || explicitPart, + _ => sameNamespace || inRootHierarchy || explicitPart, + }; + + if (includeNode && astNodeBase is not null && InheritsFrom(type, astNodeBase)) + nodes.Add(ExtractNode(type, contextBase, astNodeBase, tokenBase, secondPassEnter, secondPassExit)); } nodes.Sort((a, b) => string.CompareOrdinal(a.FullName, b.FullName)); @@ -62,28 +86,30 @@ public static GrammarModel Extract(Compilation compilation, INamedTypeSymbol roo foreach (var td in ExtractTokenDefsFromRules(n)) tokenDefs.Add(td); + // 文法で実際に参照される Token 派生型だけを検証する。Token の名前空間には依存しない。 + var usedTokenTypes = new HashSet(); + foreach (var n in nodes) + foreach (var r in n.Rules) + foreach (var p in r.Parameters) + if (p.IsToken && p.TypeFullName != TokenFullName) + usedTokenTypes.Add(p.TypeFullName); + foreach (var type in allTypes) + if (usedTokenTypes.Contains(type.ToDisplayString()) && !HasStringConstructor(type)) + tokenDerivedWarnings.Add(type.ToDisplayString()); + // [Skip] パターン ([Grammar] クラスまたはアセンブリ) を収集。 var skipPatterns = new List(); foreach (var a in rootType.GetAttributes()) - if (a.AttributeClass?.Name == "SkipAttribute" && a.ConstructorArguments.Length > 0 && a.ConstructorArguments[0].Value is string ss) + if (IsAstFirstAttribute(a, "SkipAttribute") && a.ConstructorArguments.Length > 0 && a.ConstructorArguments[0].Value is string ss) + skipPatterns.Add(ss); + foreach (var a in compilation.Assembly.GetAttributes()) + if (IsAstFirstAttribute(a, "SkipAttribute") && a.ConstructorArguments.Length > 0 && a.ConstructorArguments[0].Value is string ss) skipPatterns.Add(ss); - - // [Grammar(Mode = "...")] の Mode と [Grammar(ParseMode = ...)] の ParseMode を取得。 - // ParseMode は enum だが Generator は Runtime を直接参照しないため整数値として読む。 - string? mode = null; - var parseMode = ParseMode.Lalr; - foreach (var a in rootType.GetAttributes()) - if (a.AttributeClass?.Name == "GrammarAttribute") - foreach (var na in a.NamedArguments) - { - if (na.Key == "Mode" && na.Value.Value is string m) mode = m; - if (na.Key == "ParseMode" && na.Value.Value is int pm) parseMode = (ParseMode)pm; - } // [OnReduce]/[Enter]/[Exit] 属性付き意味解析ルール ([Grammar] ルートクラスの static メソッド) を収集。 var analyzeRules = ExtractAnalyzeRules(rootType, astNodeBase, contextBase); - return new GrammarModel(rootType.ToDisplayString(), nodes, Dedup(tokenDefs), skipPatterns, mode, rootLocation, tokenDerivedWarnings, analyzeRules, parseMode); + return new GrammarModel(rootType.ToDisplayString(), nodes, Dedup(tokenDefs), skipPatterns, mode, rootLocation, tokenDerivedWarnings, analyzeRules, parseMode, discovery); } /// [OnReduce]/[Enter]/[Exit] 属性付き意味解析ルール ([Grammar] ルートクラスの static メソッド) を収集。 @@ -132,7 +158,7 @@ private static NodeModel ExtractNode(INamedTypeSymbol type, INamedTypeSymbol? co var precAssoc = AstFirst.Core.Parsing.Associativity.Left; foreach (var a in type.GetAttributes()) { - if (a.AttributeClass?.Name != "PrecedenceAttribute") continue; + if (!IsAstFirstAttribute(a, "PrecedenceAttribute")) continue; if (a.ConstructorArguments.Length > 0 && a.ConstructorArguments[0].Value is int pr) precPriority = pr; foreach (var na in a.NamedArguments) { @@ -198,7 +224,7 @@ private static (string? regex, int priority, string? kind) GetPattern(Microsoft. { foreach (var a in symbol.GetAttributes()) { - if (a.AttributeClass?.Name is not ("PatternAttribute" or "TokenAttribute")) continue; + if (!IsAstFirstAttribute(a, "PatternAttribute") && !IsAstFirstAttribute(a, "TokenAttribute")) continue; string? regex = a.ConstructorArguments.Length > 0 && a.ConstructorArguments[0].Value is string s ? s : null; int priority = 0; string? kind = null; @@ -217,7 +243,7 @@ private static int GetRepeatMin(Microsoft.CodeAnalysis.ISymbol symbol) { foreach (var a in symbol.GetAttributes()) { - if (a.AttributeClass?.Name != "RepeatAttribute") continue; + if (!IsAstFirstAttribute(a, "RepeatAttribute")) continue; int min = 1; foreach (var na in a.NamedArguments) if (na.Key == "Min" && na.Value.Value is int m) min = m; @@ -263,7 +289,24 @@ private static IEnumerable ExtractTokenDefsFromRules(NodeModel no private static bool HasAttribute(Microsoft.CodeAnalysis.ISymbol symbol, string attrName) { foreach (var a in symbol.GetAttributes()) - if (a.AttributeClass?.Name == attrName) return true; + if (IsAstFirstAttribute(a, attrName)) return true; + return false; + } + + private static bool IsAstFirstAttribute(AttributeData attribute, string attributeName) + => attribute.AttributeClass?.ToDisplayString() == "AstFirst." + attributeName; + + private static bool IsGrammarPart(INamedTypeSymbol type, INamedTypeSymbol rootType) + { + foreach (var attribute in type.GetAttributes()) + { + if (!IsAstFirstAttribute(attribute, "GrammarPartAttribute") + || attribute.ConstructorArguments.Length == 0) + continue; + if (attribute.ConstructorArguments[0].Value is INamedTypeSymbol configuredRoot + && SymbolEqualityComparer.Default.Equals(configuredRoot, rootType)) + return true; + } return false; } @@ -280,12 +323,14 @@ private static bool InheritsFrom(ITypeSymbol type, INamedTypeSymbol baseType) return false; } - /// (string) を1つ取る public コンストラクタがあるか (G7: new DerivedType(token.Text) の生成に必要)。 + /// 生成された Parser から呼び出せる (string) コンストラクタがあるか。 private static bool HasStringConstructor(INamedTypeSymbol type) { foreach (var ctor in type.Constructors) { - if (ctor.IsStatic || ctor.DeclaredAccessibility == Accessibility.Private) continue; + if (ctor.IsStatic + || ctor.DeclaredAccessibility is not (Accessibility.Public or Accessibility.Internal or Accessibility.ProtectedOrInternal)) + continue; var parms = ctor.Parameters; if (parms.Length == 1 && parms[0].Type.SpecialType == SpecialType.System_String) return true; diff --git a/src/AstFirst.Generator/Models.cs b/src/AstFirst.Generator/Models.cs index 4d704b8..937a8f8 100644 --- a/src/AstFirst.Generator/Models.cs +++ b/src/AstFirst.Generator/Models.cs @@ -15,6 +15,14 @@ public enum ParseMode LightGlr, } +/// Runtime の AstFirst.GrammarDiscovery と値を一致させる。 +public enum GrammarDiscovery +{ + NamespaceAndTypeHierarchy, + TypeHierarchy, + Namespace, +} + /// DSL から抽出した文法モデル。等価比較可能 (IncrementalGenerator のキャッシュ判定用)。 /// シンボル/構文ノードは一切持たず、文字列/整数/bool のみ。 public sealed class GrammarModel : IEquatable @@ -22,6 +30,7 @@ public sealed class GrammarModel : IEquatable public string RootTypeFullName { get; } public string? Mode { get; } // [Grammar(Mode=...)] の Mode public ParseMode ParseMode { get; } // [Grammar(ParseMode=...)] の ParseMode (既定 Lalr) + public GrammarDiscovery Discovery { get; } public IReadOnlyList Nodes { get; } public IReadOnlyList TokenDefs { get; } public IReadOnlyList SkipPatterns { get; } @@ -36,7 +45,8 @@ public sealed class GrammarModel : IEquatable public GrammarModel(string rootTypeFullName, IReadOnlyList nodes, IReadOnlyList tokenDefs, IReadOnlyList? skipPatterns = null, string? mode = null, Location? rootLocation = null, IReadOnlyList? tokenDerivedWarnings = null, IReadOnlyList? analyzeRules = null, - ParseMode parseMode = ParseMode.Lalr) + ParseMode parseMode = ParseMode.Lalr, + GrammarDiscovery discovery = GrammarDiscovery.NamespaceAndTypeHierarchy) { RootTypeFullName = rootTypeFullName; Nodes = nodes; @@ -47,12 +57,17 @@ public GrammarModel(string rootTypeFullName, IReadOnlyList nodes, IRe TokenDerivedWarnings = tokenDerivedWarnings ?? Array.Empty(); AnalyzeRules = analyzeRules ?? Array.Empty(); ParseMode = parseMode; + Discovery = discovery; } public bool Equals(GrammarModel? other) => other is not null && RootTypeFullName == other.RootTypeFullName + && Mode == other.Mode && SeqEqual(Nodes, other.Nodes) && SeqEqual(TokenDefs, other.TokenDefs) - && SeqEqual(AnalyzeRules, other.AnalyzeRules) && ParseMode == other.ParseMode; + && SeqEqual(SkipPatterns, other.SkipPatterns) + && SeqEqual(TokenDerivedWarnings, other.TokenDerivedWarnings) + && SeqEqual(AnalyzeRules, other.AnalyzeRules) && ParseMode == other.ParseMode + && Discovery == other.Discovery; /// いずれかのノードが IOnSecondPassEnter/Exit を実装するか、[Enter]/[Exit] ルールがあるか。 /// いずれもなければ Walker/Walk を生成しない (空走査回避・ゼロコスト)。 @@ -73,8 +88,13 @@ public override int GetHashCode() { int h = StringComparer.Ordinal.GetHashCode(RootTypeFullName); for (int i = 0; i < Nodes.Count; i++) h = unchecked(h * 31 + Nodes[i].GetHashCode()); + for (int i = 0; i < TokenDefs.Count; i++) h = unchecked(h * 31 + TokenDefs[i].GetHashCode()); + for (int i = 0; i < SkipPatterns.Count; i++) h = unchecked(h * 31 + StringComparer.Ordinal.GetHashCode(SkipPatterns[i])); + if (Mode is not null) h = unchecked(h * 31 + StringComparer.Ordinal.GetHashCode(Mode)); + for (int i = 0; i < TokenDerivedWarnings.Count; i++) h = unchecked(h * 31 + StringComparer.Ordinal.GetHashCode(TokenDerivedWarnings[i])); for (int i = 0; i < AnalyzeRules.Count; i++) h = unchecked(h * 31 + AnalyzeRules[i].GetHashCode()); h = unchecked(h * 31 + (int)ParseMode); + h = unchecked(h * 31 + (int)Discovery); return h; } @@ -233,7 +253,7 @@ public bool Equals(ChildModel? other) => /// 意味解析ルールのフェーズ。 public enum AnalyzePhase { - /// 1パス目: reduce 時 (ボトムアップ)。[OnReduce] 属性。Walker 不要 (コンストラクタ経路)。 + /// 1パス目: reduce 時 (ボトムアップ)。[OnReduce] 属性。Parser の reduce 経路から呼び出す。 OnReduce, /// 2パス目: ノードに入る時 (トップダウン)。[Enter] 属性。Walker の Enter フェーズ。 Enter, diff --git a/src/AstFirst.Generator/ParserEmitter.cs b/src/AstFirst.Generator/ParserEmitter.cs index b511d7f..b6305a7 100644 --- a/src/AstFirst.Generator/ParserEmitter.cs +++ b/src/AstFirst.Generator/ParserEmitter.cs @@ -305,7 +305,7 @@ private static void EmitHelpers(StringBuilder sb, Grammar grammar, GrammarModel case ReduceActionModel action: { int len = prod.Rhs.Length; - sb.Append(" case ").Append(prod.Id).Append(": { return new ").Append(action.AstTypeName).Append("(\"").Append(action.RuleName).Append("\""); + sb.Append(" case ").Append(prod.Id).Append(": { var __node = new ").Append(action.AstTypeName).Append("(\"").Append(action.RuleName).Append("\""); for (int j = 0; j < action.Parameters.Count; j++) { sb.Append(", "); @@ -319,7 +319,9 @@ private static void EmitHelpers(StringBuilder sb, Grammar grammar, GrammarModel } else sb.Append("(").Append(p.CastTypeName).Append(")values[top - ").Append(len).Append(" + ").Append(p.ChildIndex).Append("]!"); } - sb.AppendLine("); }"); + sb.Append("); "); + EmitOnReduceAnalyzeRules(sb, model, action.AstTypeName, "__node"); + sb.AppendLine("return __node; }"); break; } case ListReduceActionModel listAction: @@ -424,7 +426,7 @@ public static string EmitPartial(GrammarModel model, NodeModel node, string ns) var typeStr = p.IsRepeat ? "System.Collections.Generic.IReadOnlyList<" + p.TypeFullName + ">" : p.TypeFullName; - sb.AppendLine(" public readonly " + typeStr + " " + prop + ";"); + sb.AppendLine(" public readonly " + typeStr + " " + prop + " = default!;"); } // RuleName: 抽象基底、または継承プロパティがない (基底が RuleName を持たない) 場合のみ生成。 if (node.Rules.Count > 0 && (isAbstractBase || !hasInherited)) @@ -510,20 +512,22 @@ public static string EmitPartial(GrammarModel model, NodeModel node, string ns) sb.AppendLine(" if (__autoSpanHas) Span = __autoSpan;"); } sb.AppendLine(" OnReduce(" + ctxCall + ");"); - // [OnReduce] 属性付き意味解析ルール ([Grammar] ルートクラスの static メソッド)。partial OnReduce の直後に呼ぶ (共存)。 - foreach (var ar in model.AnalyzeRules) - { - if (ar.Phase != AnalyzePhase.OnReduce) continue; - if (ar.TargetNodeFullName != node.FullName) continue; - if (ctxCall.Length == 0) continue; // ctx なしノードには注入不可 - sb.AppendLine(" " + ar.GrammarClassFullName + "." + ar.MethodName + "(this, (AstFirst.SemanticContext)" + ctxCall + ");"); - } sb.AppendLine(" }"); } sb.AppendLine("}"); return sb.ToString(); } + internal static void EmitOnReduceAnalyzeRules(StringBuilder sb, GrammarModel model, string nodeTypeName, string nodeVariable) + { + foreach (var rule in model.AnalyzeRules) + { + if (rule.Phase != AnalyzePhase.OnReduce || rule.TargetNodeFullName != nodeTypeName) continue; + sb.Append(rule.GrammarClassFullName).Append('.').Append(rule.MethodName) + .Append('(').Append(nodeVariable).Append(", (").Append(rule.CtxTypeFullName).Append(")ctx); "); + } + } + /// ノードの [Rule] の最後の SemanticContext 派生引数の型。なければ null (ctx なし)。 private static string? CtxTypeOf(NodeModel n) { diff --git a/src/AstFirst.Generator/ParserGenerator.cs b/src/AstFirst.Generator/ParserGenerator.cs index 96cfe31..7a73d00 100644 --- a/src/AstFirst.Generator/ParserGenerator.cs +++ b/src/AstFirst.Generator/ParserGenerator.cs @@ -19,17 +19,21 @@ public void Initialize(IncrementalGeneratorInitializationContext context) .ForAttributeWithMetadataName( GrammarAttributeFullName, predicate: (node, _) => node is ClassDeclarationSyntax, - transform: (ctx, _) => ModelExtraction.Extract(ctx)) - .Where(m => m is not null) - .Select((m, _) => m!); + transform: (ctx, _) => ModelExtraction.ExtractAll(ctx)) + .SelectMany((items, _) => items); // Lexer と Parser を生成。 context.RegisterSourceOutput(models.Collect(), (spc, modelArray) => { + var emittedModels = new HashSet(); + var emittedPartials = new HashSet(); foreach (var model in modelArray) { var (ns, typeName) = CodeEmitter.SplitFullName(model.RootTypeFullName); var suffix = string.IsNullOrEmpty(model.Mode) ? "" : "_" + model.Mode; + var modelKey = model.RootTypeFullName + "\0" + model.Mode; + if (!emittedModels.Add(modelKey)) continue; + var modelHintName = HintNamePart(modelKey); // テーブルと DFA を1回だけ構築し、Lexer/Parser の生成で共有 (重複ビルドを避ける)。 var (grammar, table) = ModelToTable.BuildWithGrammar(model); @@ -51,23 +55,41 @@ public void Initialize(IncrementalGeneratorInitializationContext context) foreach (var tdw in model.TokenDerivedWarnings) spc.ReportDiagnostic(Microsoft.CodeAnalysis.Diagnostic.Create(DiagnosticDescriptors.TokenDerivedNoStringCtor, model.RootLocation, $"Token 派生型 '{tdw}' に (string) コンストラクタがありません (new DerivedType(token.Text) の生成に必要) / Token-derived type '{tdw}' has no (string) constructor")); - spc.AddSource(typeName + suffix + "Lexer.g.cs", CodeEmitter.EmitLexer(model, dfa, rules, typeName + suffix + "Lexer", ns)); - spc.AddSource(typeName + suffix + "Parser.g.cs", model.ParseMode == ParseMode.LightGlr + spc.AddSource(modelHintName + "_Lexer.g.cs", CodeEmitter.EmitLexer(model, dfa, rules, typeName + suffix + "Lexer", ns)); + spc.AddSource(modelHintName + "_Parser.g.cs", model.ParseMode == ParseMode.LightGlr ? GlrParserEmitter.EmitParser(model, grammar, table, rules, ns) : ParserEmitter.EmitParser(model, grammar, table, rules, ns)); // 汎用 Walker: IOnSecondPassEnter/Exit または [Enter]/[Exit] を使う文法でのみ生成 (ゼロコスト)。 if (model.HasSecondPass) - spc.AddSource(typeName + suffix + "Walker.g.cs", WalkerEmitter.EmitWalker(model, ns)); + spc.AddSource(modelHintName + "_Walker.g.cs", WalkerEmitter.EmitWalker(model, ns)); // 各ノードの partial (子プロパティ + OnReduce/OnSecondPass 宣言 + partial コンストラクタ)。 // [Rule] を持つ抽象基底 (中間抽象のプロパティ宣言) は protected コンストラクタ生成のため partial 必要。 // [Rule] のない抽象クラスはフィールド/コンストラクタがないので partial 不要。 foreach (var node in model.Nodes) { if (node.IsAbstract && node.Rules.Count == 0) continue; - var simple = CodeEmitter.SplitFullName(node.FullName).type; - spc.AddSource(typeName + suffix + "_" + simple + ".partial.g.cs", ParserEmitter.EmitPartial(model, node, ns)); + var partialKey = node.FullName; + if (!emittedPartials.Add(partialKey)) continue; + var hintName = HintNamePart(node.FullName); + spc.AddSource(modelHintName + "_" + hintName + ".partial.g.cs", ParserEmitter.EmitPartial(model, node, ns)); } } }); } + + private static string HintNamePart(string fullName) + { + var chars = fullName.ToCharArray(); + uint hash = 2166136261; + unchecked + { + for (int i = 0; i < chars.Length; i++) + { + hash = (hash ^ chars[i]) * 16777619; + if (!char.IsLetterOrDigit(chars[i]) && chars[i] != '_') + chars[i] = '_'; + } + } + return new string(chars) + "_" + hash.ToString("x8"); + } } diff --git a/src/AstFirst.Runtime/Attributes.cs b/src/AstFirst.Runtime/Attributes.cs index aefa3dd..d10dc84 100644 --- a/src/AstFirst.Runtime/Attributes.cs +++ b/src/AstFirst.Runtime/Attributes.cs @@ -24,6 +24,17 @@ public enum ParseMode LightGlr, } +/// 文法に含めるノードを探索する方法。 +public enum GrammarDiscovery +{ + /// 同じ名前空間の AstNode 派生型に加え、別名前空間のルート派生型と明示的な GrammarPart を含める。 + NamespaceAndTypeHierarchy, + /// 名前空間を使わず、ルート派生型と明示的な GrammarPart だけを含める。 + TypeHierarchy, + /// 従来互換: 同じ名前空間の AstNode 派生型と明示的な GrammarPart を含める。 + Namespace, +} + /// 文法の開始記号 (ルート非終端) のクラスに付ける。Generator の抽出開始点。 [AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] public sealed class GrammarAttribute : Attribute @@ -33,6 +44,18 @@ public sealed class GrammarAttribute : Attribute /// パーサの実行モード。既定は (確定 LALR(1))。 /// は軽量 GLR (コンフリクトを並行 fork で解決)。 public ParseMode ParseMode { get; set; } = ParseMode.Lalr; + /// 文法ノードの探索方法。 + public GrammarDiscovery Discovery { get; set; } = GrammarDiscovery.NamespaceAndTypeHierarchy; +} + +/// +/// 名前空間や型階層だけでは発見できない AST ノードを、指定した文法へ明示的に参加させる。 +/// 複数文法への参加も可能。 +/// +[AttributeUsage(AttributeTargets.Class, AllowMultiple = true, Inherited = false)] +public sealed class GrammarPartAttribute(Type grammarRoot) : Attribute +{ + public Type GrammarRoot { get; } = grammarRoot; } /// スキップパターン (空白・コメント等)。クラスまたはアセンブリに付ける。 diff --git a/src/AstFirst.Runtime/ScopedSymbolTable.cs b/src/AstFirst.Runtime/ScopedSymbolTable.cs index 7c8d188..844d786 100644 --- a/src/AstFirst.Runtime/ScopedSymbolTable.cs +++ b/src/AstFirst.Runtime/ScopedSymbolTable.cs @@ -100,7 +100,7 @@ public ScopedSymbolTable() } /// キー+種類付きの子スコープを開き、それを にする。同名種類の複数スコープもキーで区別。 - public Scope PushScope(string key, ScopeKind kind) + public Scope PushScope(string? key, ScopeKind kind) { Current = new Scope(Current, Current.Depth + 1, key, kind); return Current; diff --git a/src/AstFirst/MiniLang/MiniLang.cs b/src/AstFirst/MiniLang/MiniLang.cs index 85ae6d8..0c5aed1 100644 --- a/src/AstFirst/MiniLang/MiniLang.cs +++ b/src/AstFirst/MiniLang/MiniLang.cs @@ -11,7 +11,7 @@ public abstract partial class Stmt : AstNode { } /// let x = expr; public sealed partial class LetStmt : Stmt { - public string Name { get; private set; } + public string Name { get; private set; } = ""; [Rule] public static void Let([Token(@"let", Priority = 1)] Token kw, [Token(@"[A-Za-z_]\w*")] Token nameTok, [Token(@"=")] Token eq, Expr value, [Token(@";")] Token semi) { } @@ -43,7 +43,7 @@ partial void OnReduce() public sealed partial class VarExpr : Expr { - public string Name { get; private set; } + public string Name { get; private set; } = ""; [Rule] public static void VarToken([Token(@"[A-Za-z_]\w*")] Token nameTok) { } partial void OnReduce() diff --git a/tests/AstFirst.Generator.Tests/ConflictDiagnosticTests.cs b/tests/AstFirst.Generator.Tests/ConflictDiagnosticTests.cs index e4ddb1b..1d106ab 100644 --- a/tests/AstFirst.Generator.Tests/ConflictDiagnosticTests.cs +++ b/tests/AstFirst.Generator.Tests/ConflictDiagnosticTests.cs @@ -19,18 +19,22 @@ namespace AstFirst { public abstract class AstNode { } public abstract class Token { } public abstract class SemanticContext { } - [System.AttributeUsage(System.AttributeTargets.Class)] - public class GrammarAttribute : System.Attribute { } + [System.AttributeUsage(System.AttributeTargets.Class, AllowMultiple = true)] + public class GrammarAttribute : System.Attribute { public string Mode { get; set; } } [System.AttributeUsage(System.AttributeTargets.Parameter)] public class PatternAttribute : System.Attribute { public PatternAttribute(string regex) {} } + [System.AttributeUsage(System.AttributeTargets.Class, AllowMultiple = true)] + public class GrammarPartAttribute : System.Attribute { public GrammarPartAttribute(System.Type root) {} } [System.AttributeUsage(System.AttributeTargets.Class)] public class PrecedenceAttribute : System.Attribute { public PrecedenceAttribute(int priority) {} } [System.AttributeUsage(System.AttributeTargets.Method)] public class RuleAttribute : System.Attribute { } + [System.AttributeUsage(System.AttributeTargets.Method)] + public class OnReduceAttribute : System.Attribute { } } "; - private static IReadOnlyList RunGenerator(string grammar) + private static GeneratorDriverRunResult RunGeneratorResult(string grammar) { var trusted = (string)System.AppContext.GetData("TRUSTED_PLATFORM_ASSEMBLIES")!; var refs = trusted.Split(Path.PathSeparator) @@ -42,9 +46,12 @@ public class RuleAttribute : System.Attribute { } CSharpGeneratorDriver driver = CSharpGeneratorDriver.Create(new ParserGenerator()); driver = (CSharpGeneratorDriver)driver.RunGenerators(compilation); - return driver.GetRunResult().Diagnostics; + return driver.GetRunResult(); } + private static IReadOnlyList RunGenerator(string grammar) + => RunGeneratorResult(grammar).Diagnostics; + [Fact] public void AmbiguousGrammarEmitsAstf001() { @@ -110,7 +117,7 @@ public void UndefinedNonTerminalEmitsAstf003() [AstFirst.Grammar] public abstract class Root : AstFirst.AstNode { } public class A : Root { - public A(B b) { } + [AstFirst.Rule] public static void Reduce(B b) { } } public abstract class B : AstFirst.AstNode { } "; @@ -127,7 +134,7 @@ public void TokenDerivedWithoutStringCtorEmitsAstf004() [AstFirst.Grammar] public class RootExpr : AstFirst.AstNode { } public class Num : RootExpr { - public Num([AstFirst.Pattern(""[0-9]+"")] AstFirst.Token n) { } + [AstFirst.Rule] public static void Reduce([AstFirst.Pattern(""[0-9]+"")] IntToken n) { } } public class IntToken : AstFirst.Token { public IntToken([AstFirst.Pattern(""[0-9]+"")] int n) { } @@ -136,4 +143,133 @@ public IntToken([AstFirst.Pattern(""[0-9]+"")] int n) { } var diagnostics = RunGenerator(grammar); Assert.Contains(diagnostics, d => d.Id == "ASTF004"); } + + [Fact] + public void TokenDerivedWithProtectedStringCtorEmitsAstf004() + { + var grammar = @" +[AstFirst.Grammar] +public class RootExpr : AstFirst.AstNode { } +public class Num : RootExpr { + [AstFirst.Rule] public static void Reduce([AstFirst.Pattern(""[0-9]+"")] ProtectedToken n) { } +} +public class ProtectedToken : AstFirst.Token { + protected ProtectedToken(string text) { } +} +"; + + var diagnostics = RunGenerator(grammar); + + Assert.Contains(diagnostics, d => d.Id == "ASTF004"); + } + + [Fact] + public void SameSimpleNodeNameInDifferentNamespacesUsesUniqueHintNames() + { + var grammar = @" +namespace RootNs { + [AstFirst.Grammar] + public abstract partial class Expr : AstFirst.AstNode { } +} +namespace FirstNs { + public sealed partial class Number : RootNs.Expr { + [AstFirst.Rule] public static void Reduce([AstFirst.Pattern(""1"")] AstFirst.Token n) { } + } +} +namespace SecondNs { + public sealed partial class Number : RootNs.Expr { + [AstFirst.Rule] public static void Reduce([AstFirst.Pattern(""2"")] AstFirst.Token n) { } + } +} +"; + + var result = RunGeneratorResult(grammar); + + Assert.DoesNotContain(result.Diagnostics, d => d.Severity == DiagnosticSeverity.Error); + var hintNames = result.Results.Single().GeneratedSources.Select(s => s.HintName).ToList(); + Assert.Equal(hintNames.Count, hintNames.Distinct().Count()); + Assert.Contains(hintNames, name => name.Contains("FirstNs_Number")); + Assert.Contains(hintNames, name => name.Contains("SecondNs_Number")); + } + + [Fact] + public void SameSimpleRootNameInDifferentNamespacesUsesUniqueHintNames() + { + var grammar = @" +namespace FirstNs { + [AstFirst.Grammar] + public abstract partial class Expr : AstFirst.AstNode { } +} +namespace SecondNs { + [AstFirst.Grammar] + public abstract partial class Expr : AstFirst.AstNode { } +} +"; + + var result = RunGeneratorResult(grammar); + var generated = result.Results.Single().GeneratedSources; + + Assert.DoesNotContain(result.Diagnostics, d => d.Id == "CS8785"); + Assert.Equal(4, generated.Length); + Assert.Equal(generated.Length, generated.Select(source => source.HintName).Distinct().Count()); + } + + [Fact] + public void MultipleGrammarModesGenerateEachParserWithoutDuplicatePartials() + { + var grammar = @" +[AstFirst.Grammar(Mode = ""A"")] +[AstFirst.Grammar(Mode = ""B"")] +public abstract partial class Expr : AstFirst.AstNode { } +public sealed partial class Number : Expr { + [AstFirst.Rule] public static void Reduce([AstFirst.Pattern(""[0-9]+"")] AstFirst.Token n) { } +} +"; + + var result = RunGeneratorResult(grammar); + var generated = result.Results.Single().GeneratedSources; + var sourceText = generated.Select(source => source.SourceText.ToString()).ToList(); + + Assert.DoesNotContain(result.Diagnostics, d => d.Id == "CS8785"); + Assert.Contains(sourceText, source => source.Contains("class Expr_ALexer")); + Assert.Contains(sourceText, source => source.Contains("class Expr_AParser")); + Assert.Contains(sourceText, source => source.Contains("class Expr_BLexer")); + Assert.Contains(sourceText, source => source.Contains("class Expr_BParser")); + Assert.Single(generated.Where(source => source.SourceText.ToString().Contains("partial class Number"))); + } + + [Fact] + public void GrammarPartSharedByTwoRootsEmitsOnePartialAndGrammarSpecificHooks() + { + var grammar = @" +namespace FirstNs { + [AstFirst.Grammar] + public abstract partial class Root : AstFirst.AstNode { + [AstFirst.OnReduce] public static void Analyze(Shared.Value node, AstFirst.SemanticContext ctx) { } + } +} +namespace SecondNs { + [AstFirst.Grammar] + public abstract partial class Root : AstFirst.AstNode { + [AstFirst.OnReduce] public static void Analyze(Shared.Value node, AstFirst.SemanticContext ctx) { } + } +} +namespace Shared { + [AstFirst.GrammarPart(typeof(FirstNs.Root))] + [AstFirst.GrammarPart(typeof(SecondNs.Root))] + public sealed partial class Value : AstFirst.AstNode { + [AstFirst.Rule] public static void Reduce([AstFirst.Pattern(""value"")] AstFirst.Token token, AstFirst.SemanticContext ctx) { } + } +} +"; + + var result = RunGeneratorResult(grammar); + var generated = result.Results.Single().GeneratedSources; + var sourceText = generated.Select(source => source.SourceText.ToString()).ToList(); + + Assert.DoesNotContain(result.Diagnostics, d => d.Id == "CS8785"); + Assert.Single(sourceText, source => source.Contains("partial class Value")); + Assert.Contains(sourceText, source => source.Contains("FirstNs.Root.Analyze(__node")); + Assert.Contains(sourceText, source => source.Contains("SecondNs.Root.Analyze(__node")); + } } diff --git a/tests/AstFirst.Generator.Tests/ModelExtractionTests.cs b/tests/AstFirst.Generator.Tests/ModelExtractionTests.cs index 5896f34..e68034b 100644 --- a/tests/AstFirst.Generator.Tests/ModelExtractionTests.cs +++ b/tests/AstFirst.Generator.Tests/ModelExtractionTests.cs @@ -121,6 +121,21 @@ public void ModelIsEquatable() Assert.Equal(a.GetHashCode(), b.GetHashCode()); } + [Fact] + public void ModelEqualityIncludesIncrementalGeneratorInputs() + { + var nodes = new List(); + var tokens = new List(); + var baseline = new GrammarModel("Expr", nodes, tokens); + + Assert.NotEqual(baseline, new GrammarModel("Expr", nodes, tokens, skipPatterns: new[] { "\\s+" })); + Assert.NotEqual(baseline, new GrammarModel("Expr", nodes, tokens, mode: "V2")); + Assert.NotEqual(baseline, new GrammarModel("Expr", nodes, tokens, + tokenDerivedWarnings: new[] { "MissingStringCtorToken" })); + Assert.NotEqual(baseline, new GrammarModel("Expr", nodes, tokens, + discovery: AstFirst.Generator.GrammarDiscovery.TypeHierarchy)); + } + private static GrammarModel ExtractSource(string source, string rootName) { var comp = CreateCompilation(source); @@ -145,6 +160,19 @@ public sealed class A : S { public A([Pattern(""a"")] Token t) { } } Assert.Contains(model.SkipPatterns, p => p == "\\s+"); } + [Fact] + public void AssemblySkipPatternCollected() + { + var source = @" +using AstFirst; +[assembly: Skip(""//[^\\n]*"")] +[Grammar] +public abstract class S : AstNode { } +"; + var model = ExtractSource(source, "S"); + Assert.Contains("//[^\\n]*", model.SkipPatterns); + } + [Fact] public void ModeExtracted() { @@ -166,4 +194,139 @@ public void ModeDefaultsToNull() var model = Extract(); Assert.Null(model.Mode); } + + [Fact] + public void DefaultDiscoveryIncludesRootDescendantsFromOtherNamespaces() + { + var source = @" +using AstFirst; +namespace RootNs { [Grammar] public abstract partial class Expr : AstNode { } } +namespace NodeNs { + public sealed partial class Num : RootNs.Expr { + [Rule] public static void Reduce([Token(""[0-9]+"")] Token value) { } + } +} +"; + var model = ExtractSource(source, "RootNs.Expr"); + Assert.Contains(model.Nodes, n => n.FullName == "NodeNs.Num"); + Assert.Equal(AstFirst.Generator.GrammarDiscovery.NamespaceAndTypeHierarchy, model.Discovery); + } + + [Fact] + public void TypeHierarchyDiscoveryDoesNotScanTheNamespace() + { + var source = @" +using AstFirst; +namespace Shared { + [Grammar(Discovery = AstFirst.GrammarDiscovery.TypeHierarchy)] + public abstract partial class Expr : AstNode { } + public sealed partial class Unrelated : AstNode { + [Rule] public static void Reduce([Token(""x"")] Token value) { } + } +} +namespace Nodes { + public sealed partial class Num : Shared.Expr { + [Rule] public static void Reduce([Token(""[0-9]+"")] Token value) { } + } +} +"; + var model = ExtractSource(source, "Shared.Expr"); + Assert.Contains(model.Nodes, n => n.FullName == "Nodes.Num"); + Assert.DoesNotContain(model.Nodes, n => n.FullName == "Shared.Unrelated"); + Assert.Equal(AstFirst.Generator.GrammarDiscovery.TypeHierarchy, model.Discovery); + } + + [Fact] + public void GrammarPartCanIncludeANodeOutsideNamespaceAndHierarchy() + { + var source = @" +using AstFirst; +namespace RootNs { + [Grammar(Discovery = AstFirst.GrammarDiscovery.TypeHierarchy)] + public abstract partial class Expr : AstNode { } +} +namespace SharedNodes { + [GrammarPart(typeof(RootNs.Expr))] + public sealed partial class SharedValue : AstNode { + [Rule] public static void Reduce([Token(""value"")] Token value) { } + } +} +"; + var model = ExtractSource(source, "RootNs.Expr"); + Assert.Contains(model.Nodes, n => n.FullName == "SharedNodes.SharedValue"); + } + + [Fact] + public void UnrelatedGrammarPartAttributeIsIgnored() + { + var source = @" +using AstFirst; +namespace Other { + [System.AttributeUsage(System.AttributeTargets.Class)] + public sealed class GrammarPartAttribute : System.Attribute { + public GrammarPartAttribute(System.Type grammarRoot) { } + } +} +namespace RootNs { + [Grammar(Discovery = AstFirst.GrammarDiscovery.TypeHierarchy)] + public abstract partial class Expr : AstNode { } +} +namespace SharedNodes { + [Other.GrammarPart(typeof(RootNs.Expr))] + public sealed partial class SharedValue : AstNode { + [Rule] public static void Reduce([Token(""value"")] Token value) { } + } +} +"; + + var model = ExtractSource(source, "RootNs.Expr"); + + Assert.DoesNotContain(model.Nodes, n => n.FullName == "SharedNodes.SharedValue"); + } + + [Fact] + public void UnrelatedGrammarAndAssemblySkipAttributesAreIgnored() + { + var source = @" +using AstFirst; +[assembly: Other.Skip(""not-a-lexer-pattern"")] +namespace Other { + [System.AttributeUsage(System.AttributeTargets.Class)] + public sealed class GrammarAttribute : System.Attribute { + public string Mode { get; set; } = """"; + } + [System.AttributeUsage(System.AttributeTargets.Assembly)] + public sealed class SkipAttribute : System.Attribute { + public SkipAttribute(string value) { } + } +} +[AstFirst.Grammar(Mode = ""Real"")] +[Other.Grammar(Mode = ""Fake"")] +public abstract partial class Expr : AstFirst.AstNode { } +"; + + var model = ExtractSource(source, "Expr"); + + Assert.Equal("Real", model.Mode); + Assert.Empty(model.SkipPatterns); + } + + [Fact] + public void NamespaceDiscoveryRetainsTheLegacyBoundary() + { + var source = @" +using AstFirst; +namespace RootNs { + [Grammar(Discovery = AstFirst.GrammarDiscovery.Namespace)] + public abstract partial class Expr : AstNode { } +} +namespace NodeNs { + public sealed partial class Num : RootNs.Expr { + [Rule] public static void Reduce([Token(""[0-9]+"")] Token value) { } + } +} +"; + var model = ExtractSource(source, "RootNs.Expr"); + Assert.DoesNotContain(model.Nodes, n => n.FullName == "NodeNs.Num"); + } } diff --git a/tests/AstFirst.Tests/AstFirst.Tests.csproj b/tests/AstFirst.Tests/AstFirst.Tests.csproj index bf05061..dc56827 100644 --- a/tests/AstFirst.Tests/AstFirst.Tests.csproj +++ b/tests/AstFirst.Tests/AstFirst.Tests.csproj @@ -6,6 +6,8 @@ enable enable false + + $(NoWarn);ASTF001 diff --git a/tests/AstFirst.Tests/Parsing/GrammarTests.cs b/tests/AstFirst.Tests/Parsing/GrammarTests.cs index 50655e9..e5008f9 100644 --- a/tests/AstFirst.Tests/Parsing/GrammarTests.cs +++ b/tests/AstFirst.Tests/Parsing/GrammarTests.cs @@ -125,4 +125,35 @@ public void CompleteGrammarHasNoUnreachableOrUndefined() Assert.Empty(g.UnreachableNonTerminals); Assert.Empty(g.UndefinedNonTerminals); } + + [Fact] + public void BuildReturnsAnImmutableSnapshotAndCanBeRepeated() + { + var builder = new GrammarBuilder(); + var start = builder.NonTerminal("S"); + var token = builder.Terminal("a"); + builder.Production(start, token); + + var first = builder.Build(start); + var second = builder.Build(start); + + Assert.Equal(2, first.Productions.Count); + Assert.Equal(2, second.Productions.Count); + Assert.Equal(first.Productions.Count, second.Productions.Count); + } + + [Fact] + public void ProductionCopiesItsRightHandSide() + { + var builder = new GrammarBuilder(); + var start = builder.NonTerminal("S"); + var first = builder.Terminal("a"); + var replacement = builder.Terminal("b"); + var rhs = new[] { first }; + + var production = new Production(0, start, rhs); + rhs[0] = replacement; + + Assert.Equal(first, production.Rhs[0]); + } } diff --git a/tests/AstFirst.Tests/Runtime/ScopedSymbolTableTests.cs b/tests/AstFirst.Tests/Runtime/ScopedSymbolTableTests.cs index 258f8c0..9c19204 100644 --- a/tests/AstFirst.Tests/Runtime/ScopedSymbolTableTests.cs +++ b/tests/AstFirst.Tests/Runtime/ScopedSymbolTableTests.cs @@ -202,6 +202,6 @@ public void ResolveOrError_Undeclared_AddsDiagnostic_ReturnsNull() var sym = t.ResolveOrError("missing", Span(0), bag); Assert.Null(sym); Assert.True(bag.HasErrors); - Assert.Equal(1, bag.Items.Count); + Assert.Single(bag.Items); } }