v1.0.0 - Expressions, Parsing etc (WIP)

.github/workflows/ci.yml

@@ -12,7 +12,7 @@ permissions:
   contents: read
 
 env:
-  RUSTFLAGS: -Dwarnings
+  RUSTFLAGS: -Dwarnings # Deny warnings - i.e. warnings cause build errors
 
 jobs:
   test:
@@ -31,6 +31,8 @@ jobs:
       - name: Enable type layout randomization
         run: echo RUSTFLAGS=${RUSTFLAGS}\ -Zrandomize-layout >> $GITHUB_ENV
         if: matrix.rust == 'nightly'
+      - name: Add TEST_RUST_MODE env variable
+        run: echo TEST_RUST_MODE=${{matrix.rust}} >> $GITHUB_ENV
       - run: cargo test
       - uses: actions/upload-artifact@v4
         if: matrix.rust == 'nightly' && always()
@@ -39,7 +41,7 @@ jobs:
           path: Cargo.lock
 
   msrv:
-    name: Fresh install of MSRV across self (1.61) + latest dependencies (1.68) compiles
+    name: MSRV (1.68) Compiles
     runs-on: ubuntu-latest
     timeout-minutes: 45
     steps:
@@ -61,6 +63,29 @@ jobs:
           toolchain: stable
       - run: ./style-check.sh
 
+  benches:
+    name: Benchmarks
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+      - run: ./bench_compilation.sh
+      - run: ./bench.sh
+
+  book:
+    name: Rust Book Test
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+      - run: ./book/test.sh
+
   miri:
     name: Miri
     runs-on: ubuntu-latest

.github/workflows/publish_book.yml

@@ -0,0 +1,44 @@
+name: Publish book to Github Pages
+
+on:
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: read  # to read the docs and build job
+  pages: write    # to deploy to pages
+  id-token: write # to verify the deployment originates from an appropriate source
+
+jobs:
+  # Build job - https://github.com/actions/upload-pages-artifact
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      # Sets up the repository configuration for github pages to be deployed from an action
+      - name: Setup Pages
+        uses: actions/configure-pages@v5
+      # Builds the book and outputs to book/book
+      - name: Build book
+        run: ./book/build.sh
+      # Creates a correctly compressed archive from the build HTML file, and outputs it as
+      # a `github-pages` artifact
+      - name: Upload static files as artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: book/book
+
+  # Deployment job - https://github.com/actions/deploy-pages
+  deploy:
+    runs-on: ubuntu-latest
+    needs: build
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      # Deploys the artifact `github-pages` created by the `upload-pages-artifact` job
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    // I was getting errors such as this when Rust Analyzer starts:
+    // 'PrimeCaches#5' has overflowed its stack
+    // This setting apparently avoids this https://github.com/rust-lang/rust-analyzer/issues/19698#issuecomment-2833543017
+    "rust-analyzer.cachePriming.enable": false
+}

CHANGELOG.md

@@ -1,3 +1,118 @@
+# Major Version 1.0
+
+## 1.0.0
+
+This moves preinterpret to an expression-based language, inspired by Rust, but with some twists to make writing code generation code quicker:
+* Token streams as a native feature
+* Flexible Javascript-like objects/arrays
+* New expressions such as `attempt { .. }` for trying alternatives
+
+### Variable Expansions
+
+* `#x` outputs the contents of `x`.
+* `#(x.to_group())` outputs the contents of `x` in a transparent group.
+
+### New Commands
+
+* Core commands:
+  * Creating errors:
+    * `%[].error("Error Message")` to output a compile error at the macro call site
+    * `%[_].error("Error Message")` to output a compile error at the given line
+    * `%[$token].error("Error Message")` to output a compile error at the span of the tokens
+    * `%[].assert(<condition>, <message>)` to assert the condition is true, else output a compile error at the macro call site
+    * `%[_].assert(<condition>, <message>)` to assert the condition is true, else output a compile error at the given line
+    * `%[$token].assert(<condition>, <message>)` to assert the condition is true, else output a compile error at the span of the tokens
+  * `#(x += %[...];)` to add extra tokens to a variable's stream.
+  * `#(let _ = %[...];)` interprets its arguments but then ignores any outputs.
+  * `%[...]` can be used to just output its interpreted contents. It's useful to create a stream value inside an expression.
+  * `%[...].reinterpret_as_run()` is like an `eval` command in scripting languages. It takes a stream, and runs it as a preinterpret expression block content like `run!{ ... }`. Similarly, `%[...].reinterpret_as_stream()` runs it as a stream literal, like `stream!{ ... }`. Each is pure - the reinterpreted code can't read from or write to variables, and can only return values.
+  * `None.configure_preinterpret(%{ iteration_limit: XXX })` can be used to adjust the iteration limit.
+* Expression commands:
+  * The expression block `#(let x = 123; let y = 1.0; y /= x; y + 1)` which is discussed in more detail below.
+* Control flow commands:
+  * `[!if! <expression> { ... }]` and `[!if! <expression> { ... } !elif! <expression> { ... } !else! { ... }]`
+  * `[!while! <expression> { ... }]`
+  * `[!for! <destructuring> in [ ... ] { ... }]`
+  * `[!loop! { ... }]`
+  * `[!continue!]`
+  * `[!break!]`
+* Token-stream utility commands:
+  * `<iterable>.is_empty()`
+  * `<iterable>.len()` which for streams gives the number of token trees in the token stream.
+  * `%group[...]` which wraps the tokens in a transparent group. Can be useful if using token streams as iteration sources, e.g. in `!for!`.
+  * `<any iterable>.intersperse(<separator (any value)>, <options>?)` which inserts the separator between each value from the iterator, and returns a vec. This can then be handled as a vector, embedded in a stream, or mapped with `to_string()` or `to_stream()` as required.
+  * `<stream>.split(<separator (stream)>, <options>?)` which can be used to split a stream with a given separating stream.
+  * `[countries, flags, capitals].zip()` or `%{ countries, flags, capitals }.zip()` which can be used to combine multiple streams together.
+
+### Expressions
+
+Expressions can be evaluated with `#(...)` and are also used in the `!if!` and `!while!` loop conditions.
+
+Expressions behave intuitively as you'd expect from writing regular rust code, except they are executed at compile time.
+
+The `#(...)` expression block behaves much like a `{ .. }` block in rust. It supports multiple statements ending with `;` and optionally a final statement.
+
+Statements are either expressions `EXPR` or `let x = EXPR`, `x = EXPR`, `x += EXPR` for some operator such as `+`.
+
+Assigment:
+* `let <pattern> = <value>` which supports patterns include ignore (`_`), array destructurings (`[a, b, ..]`), object destructurings `{ a: x, b, .. }`, and stream parsing `%[..]`.
+
+The following are recognized values:
+* Object literals `%{ x: "Hello", y, ["z"]: "World" }` behave similarly to Javascript objects.
+* Token stream literals `%[...]` take any token stream, and support embedding `#variables` or `#(<..expressions..>)` inside them.
+* Raw token stream literals `%raw[...]` are used to capture raw tokens, and are not interpreted (e.g. `#` has no special meaning).
+* Integer literals, with or without a suffix
+* Float literals, with or without a suffix
+* Boolean literals
+* String literals
+* Char literals
+* Other literals
+* Rust [ranges](https://doc.rust-lang.org/reference/expressions/range-expr.html)
+* Token streams which are defined as `[...]`.
+
+The following operators are supported:
+* The numeric operators: `+ - * / % & | ^`
+* The lazy boolean operators: `|| &&`
+* The comparison operators: `== != < > <= >=`
+* The shift operators: `>> <<`
+* The concatenation operator: `+` can be used to concatenate strings and streams.
+* Casting with `as` including to untyped integers/floats with `as int` and `as float` and to a flattened stream with `as stream`.
+* () and none-delimited groups for precedence
+
+The following methods are supported:
+* On all values:
+  * `.clone()` - converts a reference to a mutable value. You will be told in an error if this is needed.
+  * `.as_mut()` - converts an owned value to a mutable value. You will be told in an error if this is needed.
+  * `.take()` - takes the value from a mutable reference, and replaces it with `None`. Useful instead of cloning.
+  * `.debug()` - a debugging aid whilst writing code. Causes a compile error with the content of the value. Roughly equivalent to `%[_].error(x.to_debug_string())`
+  * `.to_debug_string()` - returns the value's contents as a string for debugging purposes
+* On arrays: `len()` and `push()`
+* On streams: `len()`
+
+An expression also supports embedding commands `[!xxx! ...]`, other expression blocks, variables and flattened variables. The value type outputted by a command depends on the command.
+
+### Transforming
+
+Transforming performs parsing of a token stream, whilst also outputting a stream. The input stream must be parsed in its entirety.
+
+Transform streams (or substreams) can be redirected to set variables or append to variables. Commands can also be injected to add to the output.
+
+Inside a transform stream, the following grammar is supported:
+
+* `@(...)`, `@(#x = ...)`, `@(#x += ...)` and `@(_ = ...)` - Explicit transform (sub)streams which either output, set, append or discard its output.
+* Explicit punctuation, idents, literals and groups. These aren't output by default, except directly inside a `@[EXACT ...]` transformer.
+
+* Named destructurings:
+  * `@IDENT` - Consumes and output any ident.
+  * `@PUNCT` - Consumes and outputs any punctation
+  * `@LITERAL` - Consumes and outputs any literal
+  * `@REST` - Consumes the rest of the input, until the end of the stream or content of the current group
+  * `@[UNTIL x]` - Consumes the rest of the input, until the end of stream OR until token `x`. `x` can be a group like `()` which matches the opening bracket `(`. 
+  * `@[GROUP ...]` - Consumes a none-delimited group. Its arguments are used to transform the group's contents.
+  * `@[EXACT(%[..])]` - Takes an input stream expression. Expects to consume exactly that stream from the output (ignoring none-groups). It outputs the parsed stream.
+* Commands: Their output is appended to the transform's output. Useful patterns include:
+  * If `@(inner = ...)` then `inner.to_group()` - wraps the output in a transparent group
+
 # Major Version 0.2
 
 ## 0.2.1

CLAUDE.md

@@ -0,0 +1,71 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+Preinterpret is a Rust procedural macro crate that provides macros which form a code generation toolkit that simplifies declarative macro development. It combines functionality from quote, paste, and syn crates to enable:
+
+- Variable definition and substitution with `#(let var = %[...];)` and `#var`
+- Commands for concatenation, case conversion, and token manipulation like `[!ident! ...]`, `[!string! ...]`, `[!ident_snake! ...]`
+- Control flow and parsing capabilities
+
+## Plans and Tasks
+
+There are various files in the `./plans` folder, the "current vision" is `2025-09-vision.md` and the "todo list" is `TODO.md`.
+
+## Architecture
+
+The codebase is organized into several main modules:
+
+- **src/lib.rs** - Main entry point and public API
+- **src/interpretation/** - Core interpreter logic for processing commands and variables
+  - `interpreter.rs` - Main interpreter implementation
+  - `commands/` - Built-in command implementations (concat, transforming, control flow, etc.)
+  - `bindings.rs` - Variable binding management
+- **src/expressions/** - Expression evaluation system for advanced features
+  - `evaluation/` - Expression evaluator with type resolution and frame management
+- **src/transformation/** - Token stream transformation utilities
+- **src/extensions/** - Helper traits and utilities for working with proc-macro2 tokens
+
+## Development Commands
+
+### Testing
+- `cargo test` - Run all tests
+- `cargo test --release` - Run tests in release mode
+- `cargo miri test` - Run tests with Miri for memory safety
+
+### Code Quality
+- `./style-check.sh` - Check formatting and run clippy (equivalent to CI checks)
+- `./style-fix.sh` - Auto-fix formatting and clippy issues
+- `cargo fmt --check` - Check formatting only
+- `cargo clippy --tests` - Run clippy lints
+
+### Build & Check
+- `cargo build` - Build the crate
+- `cargo check` - Quick syntax/type check
+- `./local-check-msrv.sh` - Test minimum supported Rust version (1.63)
+
+### Documentation
+- `cargo doc --open` - Build and open documentation
+- `./book/test.sh` - Build the mdbook documentation
+
+## Key Implementation Notes
+
+This is a proc-macro crate (`proc-macro = true` in Cargo.toml) that:
+
+- Uses syn 2.0 for parsing with full feature set enabled
+- Supports minimum Rust version 1.63 (Edition 2021)
+- Employs RefCell/Rc patterns for interior mutability in expression evaluation
+- Uses trybuild for compile-time testing of macro expansions
+
+The expression system (newer feature) provides advanced capabilities like mathematical operations and control flow, built on top of the core interpretation framework.
+
+## Testing Strategy
+
+Tests are located in the `tests/` directory and use trybuild for testing compilation failures. The CI runs tests on stable, beta, and nightly Rust versions with warnings treated as errors.
+
+## Commit Strategy
+
+* Run `style-fix.sh` before committing
+* Use the conventional commits pattern for commit message prefixes.

Cargo.toml

@@ -11,14 +11,29 @@ keywords = ["macros", "declarative-macros", "toolkit", "interpreter", "preproces
 # And whilst there's a development-tools::procedural-macro-helpers, there's no declarative macro category.
 categories = ["development-tools", "compilers"]
 # MSRV 1.56.0 is the start of Edition 2021
-# MSRV 1.61.0 was the old MSRV of syn
-# MRSV 1.68.0 is the new latest MSRV of syn (as of syn 2.0.107 released on 19th October 2025) - but keeping on MSRV 1.61.0 for now
+# MSRV 1.61.0 is the old MSRV of syn
+# MRSV 1.63.0 is needed to support RefMut::filter_map
+# MRSV 1.68.0 is the new latest MSRV of syn (as of syn 2.0.107 released on 19th October 2025)
 # If changing this, also update the local-check-msrv.sh script and ci.yml
-rust-version = "1.61"
+rust-version = "1.68"
 
 [lib]
 proc-macro = true
 
+[[bench]]
+name = "basic"
+harness = false
+required-features = ["benchmark"]
+
+[features]
+benchmark = [] # For internal use only
+debug = [] # Non-stable, for internal use only
+
 [dependencies]
-proc-macro2 = { version = "1.0" }
-syn = { version = "2.0", default-features = false, features = ["parsing"] }
+proc-macro2 = { version = "1.0.93" }
+syn = { version = "2.0.107", default-features = false, features = ["parsing", "derive", "printing", "clone-impls", "full"] }
+quote = { version = "1.0.38", default-features = false }
+slotmap = { version = "1.0.7" }
+
+[dev-dependencies]
+trybuild = { version = "1.0.110", features = ["diff"] }

KNOWN_ISSUES.md

@@ -0,0 +1,7 @@
+# Since Major Version 0.3.0
+
+* `[MAX LITERAL]` - In an expression, the maximum negative integer literal is not valid, for example `-128i8`.
+  * By comparison, in `rustc`, there is special handling for such literals, so that e.g. `-128i8` and `--(-128i8)` are accepted.
+  * In rust analyzer, before the full pass, it seems to store literals as u128 and perform wrapping arithmetic on them. On a save / full pass, it does appear to reject literals which are out of bounds (e.g. 150i8).
+  * We could fix this by special casing maximal signed integer literals (e.g. 128 for an i8) which have yet to be involved in a non-unary expression.
+