diff --git a/.github/workflows/rust.yml b/.github/workflows/ci.yml similarity index 84% rename from .github/workflows/rust.yml rename to .github/workflows/ci.yml index 59bb478..b1dd93d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/ci.yml @@ -1,12 +1,15 @@ -# CI pipeline for the Rust implementation +--- +# yamllint disable rule:truthy rule:line-length +# CI pipeline for PRs (lint, test, coverage, sonar) and tag release builds. name: CI Pipeline on: - push: + pull_request: branches: - main - master - develop + push: tags: - "v*" @@ -17,7 +20,6 @@ jobs: dictionaries: name: Prepare FIX specs runs-on: ubuntu-latest - # Specs are needed for all flows. steps: - uses: actions/checkout@v4 - name: Download FIX XML dictionaries @@ -29,119 +31,18 @@ jobs: name: fix-specs path: resources - build-matrix: - name: Build release (tags only) - strategy: - fail-fast: true - matrix: - include: - - os: macos-latest - target: aarch64-apple-darwin - artifact_path: target/aarch64-apple-darwin/release/fixdecoder - asset_suffix: darwin-arm64 - ext: "" - - os: macos-15-intel - target: x86_64-apple-darwin - artifact_path: target/x86_64-apple-darwin/release/fixdecoder - asset_suffix: darwin-x86_64 - ext: "" - - os: windows-latest - target: x86_64-pc-windows-msvc - artifact_path: target/x86_64-pc-windows-msvc/release/fixdecoder.exe - asset_suffix: windows-x86_64 - ext: ".exe" - - os: ubuntu-latest - target: x86_64-unknown-linux-gnu - artifact_path: target/x86_64-unknown-linux-gnu/release/fixdecoder - asset_suffix: linux-gnu-x86_64 - ext: "" - - os: ubuntu-latest - target: x86_64-unknown-linux-musl - setup: sudo apt-get update && sudo apt-get install -y musl-tools - artifact_path: target/x86_64-unknown-linux-musl/release/fixdecoder - asset_suffix: linux-musl-x86_64 - ext: "" - runs-on: ${{ matrix.os }} - needs: [dictionaries] - # Only build release artifacts when tagging; skips lint/coverage. - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - env: - FIXDECODER_BRANCH: ${{ github.ref_name }} - FIXDECODER_COMMIT: ${{ github.sha }} - FIXDECODER_GIT_URL: https://github.com/${{ github.repository }} - steps: - - uses: actions/checkout@v4 - - name: Compute lockfile hash - id: lockfile-hash - shell: bash - run: | - if command -v python3 >/dev/null 2>&1; then - python3 ci/compute_lockfile_hash.py Cargo.lock - elif command -v python >/dev/null 2>&1; then - python ci/compute_lockfile_hash.py Cargo.lock - else - echo "Python is required to compute the lockfile hash." >&2 - exit 1 - fi - - name: Set up Rust - uses: dtolnay/rust-toolchain@stable - with: - components: clippy,rustfmt,llvm-tools-preview - targets: ${{ matrix.target }} - - name: Download FIX specs artifact - uses: actions/download-artifact@v4 - with: - name: fix-specs - path: resources - - name: Cache cargo registry and target - uses: actions/cache@v4 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-${{ matrix.target }}-cargo-${{ steps.lockfile-hash.outputs.hash }} - restore-keys: | - ${{ runner.os }}-${{ matrix.target }}-cargo- - ${{ runner.os }}-cargo- - - name: Install target dependencies - if: ${{ matrix.setup != '' }} - run: ${{ matrix.setup }} - - name: Prepare build inputs - shell: bash - run: | - source ci/ci_helper.sh - ensure_build_metadata - cargo run --quiet --bin generate_sensitive_tags >/dev/null - - name: Build release - shell: bash - run: | - if [[ "${{ matrix.target }}" == "x86_64-unknown-linux-musl" ]]; then - export CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-C target-feature=+crt-static" - fi - cargo fmt --all - cargo build --release --locked --target ${{ matrix.target }} - env: - RUSTFLAGS: "" - - name: Stage artifact - shell: bash - run: | - mkdir -p dist - cp "${{ matrix.artifact_path }}" "dist/fixdecoder-${{ matrix.asset_suffix }}${{ matrix.ext }}" - - name: Upload artifact - uses: actions/upload-artifact@v4 - with: - name: fixdecoder-${{ matrix.asset_suffix }} - path: dist/fixdecoder-${{ matrix.asset_suffix }}${{ matrix.ext }} - quality: name: Lint & Coverage runs-on: ubuntu-latest needs: dictionaries - # Run on branch pushes for fast feedback (debug build + coverage); skip tags. - if: github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/') + if: github.event_name == 'pull_request' steps: - uses: actions/checkout@v4 + - name: Lint workflow YAML + run: | + python3 -m pip install --upgrade pip + python3 -m pip install yamllint + yamllint .github/workflows - name: Compute lockfile hash id: lockfile-hash shell: bash @@ -225,8 +126,7 @@ jobs: name: Sonar (coverage import) runs-on: ubuntu-latest needs: [quality] - # Run for branch pushes (not tags) when a token is available. - if: github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/') + if: github.event_name == 'pull_request' steps: - uses: actions/checkout@v4 - name: Determine toolchain channel @@ -240,6 +140,9 @@ jobs: toolchain=$(head -n1 rust-toolchain | tr -d '[:space:]') fi echo "toolchain=$toolchain" >> "$GITHUB_OUTPUT" + - name: Ensure cache directories exist + run: | + mkdir -p "$HOME/.cargo/bin" "$HOME/.rustup" - name: Compute lockfile hash id: lockfile-hash run: echo "hash=$(sha256sum Cargo.lock | cut -d ' ' -f1)" >> "$GITHUB_OUTPUT" @@ -287,11 +190,135 @@ jobs: echo "[]" > target/clippy.json fi CLIPPY_REPORT="$(pwd)/target/clippy.json" + PR_OPTS=() + if [ "${{ github.event_name }}" = "pull_request" ]; then + PR_OPTS+=("-Dsonar.pullrequest.key=${{ github.event.pull_request.number }}") + PR_OPTS+=("-Dsonar.pullrequest.branch=${{ github.event.pull_request.head.ref }}") + PR_OPTS+=("-Dsonar.pullrequest.base=${{ github.event.pull_request.base.ref }}") + PR_OPTS+=("-Dsonar.pullrequest.provider=github") + fi sonar-scanner \ -Dsonar.host.url=https://sonarcloud.io \ -Dsonar.externalIssuesReportPaths=target/coverage/sonar-generic-issues.json \ -Dsonar.rust.clippy.reportPaths="${CLIPPY_REPORT}" \ - -Dsonar.rust.clippy.enabled=false + -Dsonar.rust.clippy.enabled=false \ + "${PR_OPTS[@]}" + + build-matrix: + name: Build release (tags only) + strategy: + fail-fast: true + matrix: + include: + - os: macos-latest + target: aarch64-apple-darwin + artifact_path: target/aarch64-apple-darwin/release/fixdecoder + asset_suffix: darwin-arm64 + ext: "" + - os: macos-15-intel + target: x86_64-apple-darwin + artifact_path: target/x86_64-apple-darwin/release/fixdecoder + asset_suffix: darwin-x86_64 + ext: "" + - os: windows-latest + target: x86_64-pc-windows-msvc + artifact_path: target/x86_64-pc-windows-msvc/release/fixdecoder.exe + asset_suffix: windows-x86_64 + ext: ".exe" + - os: ubuntu-latest + target: x86_64-unknown-linux-gnu + artifact_path: target/x86_64-unknown-linux-gnu/release/fixdecoder + asset_suffix: linux-gnu-x86_64 + ext: "" + - os: ubuntu-latest + target: x86_64-unknown-linux-musl + setup: sudo apt-get update && sudo apt-get install -y musl-tools + artifact_path: target/x86_64-unknown-linux-musl/release/fixdecoder + asset_suffix: linux-musl-x86_64 + ext: "" + runs-on: ${{ matrix.os }} + needs: [dictionaries] + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + env: + FIXDECODER_BRANCH: ${{ github.ref_name }} + FIXDECODER_COMMIT: ${{ github.sha }} + FIXDECODER_GIT_URL: https://github.com/${{ github.repository }} + steps: + - uses: actions/checkout@v4 + - name: Compute lockfile hash + id: lockfile-hash + shell: bash + run: | + if command -v python3 >/dev/null 2>&1; then + python3 ci/compute_lockfile_hash.py Cargo.lock + elif command -v python >/dev/null 2>&1; then + python ci/compute_lockfile_hash.py Cargo.lock + else + echo "Python is required to compute the lockfile hash." >&2 + exit 1 + fi + - name: Set up Rust + uses: dtolnay/rust-toolchain@stable + with: + components: clippy,rustfmt,llvm-tools-preview + targets: ${{ matrix.target }} + - name: Download FIX specs artifact + uses: actions/download-artifact@v4 + with: + name: fix-specs + path: resources + - name: Cache cargo registry and target + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-${{ matrix.target }}-cargo-${{ steps.lockfile-hash.outputs.hash }} + restore-keys: | + ${{ runner.os }}-${{ matrix.target }}-cargo- + ${{ runner.os }}-cargo- + - name: Install target dependencies + if: ${{ matrix.setup != '' }} + run: ${{ matrix.setup }} + - name: Prepare build inputs + shell: bash + run: | + source ci/ci_helper.sh + ensure_build_metadata + cargo run --quiet --bin generate_sensitive_tags >/dev/null + - name: Build release + shell: bash + run: | + if [[ "${{ matrix.target }}" == "x86_64-unknown-linux-musl" ]]; then + export CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-C target-feature=+crt-static" + fi + cargo fmt --all + cargo build --release --locked --workspace --target ${{ matrix.target }} + env: + RUSTFLAGS: "" + - name: Stage artifact + shell: bash + run: | + mkdir -p dist + cp "${{ matrix.artifact_path }}" "dist/fixdecoder-${{ matrix.asset_suffix }}${{ matrix.ext }}" + pcap_path="target/${{ matrix.target }}/release/pcap2fix${{ matrix.ext }}" + if [ -f "$pcap_path" ]; then + cp "$pcap_path" "dist/pcap2fix-${{ matrix.asset_suffix }}${{ matrix.ext }}" + else + echo "pcap2fix binary not found at $pcap_path" >&2 + exit 1 + fi + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: fixdecoder-${{ matrix.asset_suffix }} + path: dist/fixdecoder-${{ matrix.asset_suffix }}${{ matrix.ext }} + - name: Upload pcap2fix artifact + uses: actions/upload-artifact@v4 + with: + name: pcap2fix-${{ matrix.asset_suffix }} + path: dist/pcap2fix-${{ matrix.asset_suffix }}${{ matrix.ext }} release: name: Publish Release @@ -321,24 +348,26 @@ jobs: - name: Download build artifacts uses: actions/download-artifact@v4 with: - pattern: fixdecoder-* + pattern: "*" path: dist merge-multiple: true - name: Rename artifacts with version run: | ver="${{ steps.version.outputs.version }}" shopt -s nullglob - for f in dist/fixdecoder-*; do + for f in dist/fixdecoder-* dist/pcap2fix-*; do base="${f##*/}" case "$base" in - fixdecoder-*.exe) - suffix="${base#fixdecoder-}" + fixdecoder-*.exe|pcap2fix-*.exe) + prefix="${base%%-*}" + suffix="${base#${prefix}-}" suffix="${suffix%.exe}" - mv "$f" "dist/fixdecoder-${ver}.${suffix}.exe" + mv "$f" "dist/${prefix}-${ver}.${suffix}.exe" ;; - fixdecoder-*) - suffix="${base#fixdecoder-}" - mv "$f" "dist/fixdecoder-${ver}.${suffix}" + fixdecoder-*|pcap2fix-*) + prefix="${base%%-*}" + suffix="${base#${prefix}-}" + mv "$f" "dist/${prefix}-${ver}.${suffix}" ;; esac done diff --git a/Cargo.lock b/Cargo.lock index 6fe0709..2aa74f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -76,6 +76,12 @@ version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "assert_cmd" version = "2.1.1" @@ -164,6 +170,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "circular" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fc239e0f6cb375d2402d48afb92f76f5404fd1df208a41930ec81eda078bea" + [[package]] name = "clap" version = "4.5.53" @@ -286,6 +298,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "etherparse" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21696e6dfe1057a166a042c6d27b89a46aad2ee1003e6e1e03c49d54fd3270d7" +dependencies = [ + "arrayvec", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -300,7 +321,7 @@ checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" [[package]] name = "fixdecoder" -version = "0.2.1" +version = "0.3.0" dependencies = [ "anyhow", "assert_cmd", @@ -410,6 +431,12 @@ version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "nix" version = "0.30.1" @@ -422,6 +449,16 @@ dependencies = [ "libc", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "normalize-line-endings" version = "0.3.0" @@ -464,6 +501,30 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "pcap-parser" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b79dfb40aef938ed2082c9ae9443f4eba21b79c1a9d6cfa071f5c2bd8d829491" +dependencies = [ + "circular", + "nom", + "rusticata-macros", +] + +[[package]] +name = "pcap2fix" +version = "0.1.0" +dependencies = [ + "anyhow", + "assert_cmd", + "clap", + "etherparse", + "pcap-parser", + "predicates", + "thiserror", +] + [[package]] name = "predicates" version = "3.1.3" @@ -595,6 +656,15 @@ dependencies = [ "semver", ] +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + [[package]] name = "rustix" version = "1.1.2" @@ -702,6 +772,26 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "unicode-ident" version = "1.0.22" diff --git a/Cargo.toml b/Cargo.toml index ec03add..92bb7cc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fixdecoder" -version = "0.2.1" +version = "0.3.0" edition = "2024" [dependencies] @@ -23,3 +23,7 @@ tempfile = "3.10" [build-dependencies] rustc_version = "0.4" + +[workspace] +members = [".", "pcap2fix"] +resolver = "2" diff --git a/Makefile b/Makefile index d4672c1..b2180f6 100644 --- a/Makefile +++ b/Makefile @@ -13,28 +13,36 @@ prepare: @cargo run --quiet --bin generate_sensitive_tags >/dev/null build: prepare - @bash -lc 'source $(CI_SCRIPT) && ensure_build_metadata && cargo fmt --all && cargo build' + @bash -lc 'source $(CI_SCRIPT) && ensure_build_metadata && cargo fmt --all && cargo build --workspace' build-release: prepare - @bash -lc 'source $(CI_SCRIPT) && ensure_build_metadata && cargo fmt --all && cargo build --release' - @python3 ci/update_readme.py - -.PHONY: update-readme -update-readme: - @python3 ci/update_readme.py + @bash -lc 'source $(CI_SCRIPT) && ensure_build_metadata && cargo fmt --all && cargo build --workspace --release' scan: prepare @bash -lc '\ source $(CI_SCRIPT) && \ ensure_build_metadata && \ cargo fmt --all --check && \ - cargo clippy --all-targets -- -D warnings && \ + cargo clippy --workspace --all-targets -- -D warnings && \ + if command -v yamllint >/dev/null 2>&1; then \ + yamllint .github/workflows || true; \ + else \ + echo "yamllint not installed; skipping YAML lint"; \ + fi; \ mkdir -p target/coverage && \ if command -v cargo-audit >/dev/null 2>&1; then \ echo "Running cargo-audit (text output)"; \ - cargo audit || true; \ + if [ -d "$${HOME}/.cargo/advisory-db" ]; then \ + cargo audit --no-fetch || true; \ + else \ + cargo audit || true; \ + fi; \ echo "Running cargo-audit (JSON) → target/coverage/rustsec.json"; \ - cargo audit --json > target/coverage/rustsec.json || true; \ + if [ -d "$${HOME}/.cargo/advisory-db" ]; then \ + cargo audit --no-fetch --json > target/coverage/rustsec.json || true; \ + else \ + cargo audit --json > target/coverage/rustsec.json || true; \ + fi; \ echo "Converting RustSec report to Sonar generic issues (target/coverage/sonar-generic-issues.json)"; \ python3 ci/convert_rustsec_to_sonar.py target/coverage/rustsec.json target/coverage/sonar-generic-issues.json || true; \ else \ @@ -48,7 +56,10 @@ coverage: build ensure_build_metadata && \ mkdir -p target/coverage && \ cargo llvm-cov clean --workspace >/dev/null 2>&1 || true; \ - cargo llvm-cov --workspace --cobertura \ + cargo llvm-cov \ + --package fixdecoder \ + --package pcap2fix \ + --cobertura \ --ignore-filename-regex "src/fix/sensitive.rs|src/bin/generate_sensitive_tags.rs" \ --output-path target/coverage/coverage.xml \ ' diff --git a/README.md b/README.md index ab931ca..e7a9156 100644 --- a/README.md +++ b/README.md @@ -44,126 +44,101 @@ I have written utilities like this in past in Java, Python, C, C++, [go](https:/ --- -# How to use it +## What is it -The utility behaves like the `cat` utility in `Linux`, except as it reads the input (either piped in from `stdin` or from a filename specified on the commandline) it scans each line for `FIX protocol` messages and prints them out highlighted in bold white while the rest of the line will be in a mid grey colour. After the line is output it will be followed by a detailed breakdown of all the `FIX Protocol` tags that were found in the message. The detailed output will use the appropriate `FIX` dictionary for the version of `FIX` specified in `BeginString (tag 8)` tag. It will also look at `DefaultApplVerID (tag 1137)` when `8=FIXT.1.1` is detected in the message. +fixdecoder is a FIX-aware “tail-like” tool and dictionary explorer. It reads from stdin or multiple log files, detects and prettifies FIX messages in stream, and fits naturally into pipelines. Each highlighted message is followed by a detailed tag breakdown using the correct dictionary for BeginString (8) (or DefaultApplVerID (1137) when 8=FIXT.1.1). It can validate on the fly (`--validate`), reporting protocol issues as it decodes, and track order state with summaries (`--summary`). For lookups, `--info` shows available/overridden dictionaries, and `--message`, `--component`, or `--tag` inspect definitions in the selected FIX version (`--fix` or default) without a live decode. -## Running the utility +## Quick start - ```bash -fixdecoder v0.2.1 (branch:develop, commit:02b044e) [rust:1.91.1] -FIX protocol utility - Dictionary lookup, file decoder, validator & prettifier +# Stream and prettify stdin (pipeline-friendly) +cat fixlog.txt | fixdecoder -Usage: fixdecoder [OPTIONS] [FILE]... +# Stream with validation + order summaries +cat fixlog.txt | fixdecoder --validate --summary +``` + + -Arguments: - [FILE]... +## Running the fixdecoder utility -Options: - --fix FIX version to use [default: 44] - --xml Path to alternative FIX XML dictionary (repeatable) - --message [] FIX Message name or MsgType (omit value to list all) - --component [] FIX Component to display (omit value to list all) - --tag [] FIX Tag number to display (omit value to list all) - --column Display enums in columns - --header Include Header block - --trailer Include Trailer block - --verbose Show full message structure with enums - --info Show schema summary - --secret Obfuscate sensitive FIX tag values - --validate Validate FIX messages during decoding - --colour [] Force coloured output - --delimiter Display delimiter between FIX fields (default: SOH) - --version Print version information and exit - --summary Track order state across messages and print a summary - -f, --follow Stream input like tail -f - -h, --help Print help +You can run fixdecoder anywhere you can run a Rust binary — no extra OS dependencies or runtime services are required. It ships with a full set of embedded FIX dictionaries. The sections below cover the key options for selecting and browsing dictionaries, controlling output/formatting, and adjusting processing modes. -Command line option examples: +## Key options at a glance - FIX dictionary lookup +- Dictionaries: `--xml`, `--fix`, `--info`, `--message`, `--component`, `--tag` +- Output/layout: `--column`, `--verbose`, `--header`, `--trailer`, `--colour`, `--delimiter` +- Processing modes: `--follow`, `--validate`, `--secret`, `--summary` - Query FIX dictionary contents by FIX Message Name or MsgType: +### `--xml` - fixdecoder [[--fix=44] [--xml=FILE --xml=FILE2 ...]] [--message[=NAME|MSGTYPE] [--verbose] [--column] [--header] [--trailer] +The `--xml` flag lets you load custom FIX dictionaries from XML files; you can pass it multiple times to register several custom dictionaries. Each file is parsed, normalised to a canonical key (e.g., FIX44, FIX50SP2), and has FIXT11 session header/trailer injected for 5.0+ if missing. Custom entries are registered for tag lookup and schema loading; they override built-ins for the same key and replace earlier `--xml` files for that key, with warnings emitted in both cases. - $ fixdecoder --message=NewOrderSingle --verbose --column --header --trailer - $ fixdecoder --message=D --verbose --column --header --trailer - - Query FIX dictionary contents by FIX Tag number: +The XML dictionaries can be downloaded from the [QuickFIX GitHub Repo](https://github.com/quickfix/quickfix/tree/master/spec) - fixdecoder [[--fix=44] [--xml=FILE --xml=FILE2 ...]] [--tag[=TAG] [--verbose] [--column] +### `--fix` - $ fixdecoder --tag=44 --verbose --column - - Query FIX dictionary contents by FIX Component Name: +The `--fix` option allows you to specify the default FIX dictionary. This defaults to FIX 4.4 (`44`). It accepts either just the version digits (e.g., `44`, `4.4`) or the same value prefixed with FIX/fix (e.g., `FIX44`, `fix4.4`). The parser normalises your input by stripping dots, uppercasing, and adding FIX if it’s missing; it then checks that key against built‑ins (`FIX27`…`FIXT11`) and any custom `--xml` overrides. If the normalised key isn’t known, it errors. - fixdecoder [[--fix=44] [--xml=FILE --xml=FILE2 ...]] [--component[=NAME] [--verbose] [--column] +### `--info` - $ fixdecoder --component=Instrument --verbose --column +`--info` is an informational mode: it prints the list of available FIX dictionary keys (built-ins plus any loaded via `--xml`), then a table of loaded dictionaries with counts and their source (built-in vs file path). The table highlights the currently selected/default FIX version (from `--fix` or the default `44`) with a leading `*` so you can see which dictionary will be used. It does not decode messages or print schema details; it’s meant to verify which dictionaries are present, which ones are being overridden by custom XML, and which version is active. - Show summary information about available FIX dictionaries: +![--info](docs/info_command.png) - fixdecoder [[--fix=44] [--xml=FILE --xml=FILE2 ...]] [--info] +## Querying the FIX dictionaries `--message`, `--component` and `--tag` - $ fixdecoder --info +Use these flags to explore the active FIX dictionary. `--verbose` adds detail / metadata, `--column` uses a compact table layout. `--header`/`--trailer` only apply to `--message` and `--component` (not `--tag`). - Prettify FIX log files with optional validation and obfuscation; if output is piped then colour is disabled by - default but can be forced on with --colour=yes: +### `--message[=]` - fixdecoder [--xml=FILE --xml=FILE2 ...] [--validate] [--colour=yes|no] [--secret] [--summary] [--follow] [--fix=VER] [--delimiter=CHAR] [file1.log file2.log ...] +Browse messages. With no value, list all message types (use --`column` for a compact view). With a name or MsgType (e.g., `D` or `NewOrderSingle`), render the message structure (fields, components, repeating groups); `--header`/`--trailer` include session blocks. Reports “Message not found” if absent. - Validate and Obfuscate a FIX logfile. +### `--component[=]` - $ fixdecoder --validate --secret logs/fix.log +Browse components. With no value, list all components (or use `--column`). With a name, render that component’s fields, nested components, and repeating groups. Reports “Component not found” if absent. - Decode all the NewOrderSingle messages in a FIX logfile and output the fix messages using a custom delimiter - also force colour mode because this example pipes the output into less. Normally colour mode is turned off - when piping the output due to the output containing ANSI control chars which may mess up processing further - down the pipe chain. +### `--tag[=]` - $ grep '35=D' logs/fix.log | fixdecoder --colour=yes --delimiter='|' | less +Browse fields. With no value, list all tags (or use `--column`). With a tag number, show that field’s details (name, type, enums, etc.). Reports “Tag not found” if absent. - Force the decoding of a FIX log to use the FIX 4.4 dictionary. Only uses the version of the FIX dictionary - specified in the FIX message header if the tag being processed is not defined in the override dictionary. - for example FIX 4.4 does not have the FIX 4.2 tag 20 (ExecTransType) +### `--validate` - $ fixdecoder --fix=44 trades.log +Validate each decoded FIX message against the active dictionary (honours `--fix` and any `--xml` overrides). Checks MsgType, BodyLength, checksum, required fields, enum/type correctness, field ordering, repeating-group structure, and duplicate disallowed tags. Validation runs alongside prettified output; any errors are appended after the message. It doesn’t stop the stream—use it to flag protocol issues while decoding - Process a FIX log file and display an order summary for each order that is processed. +### `--secret` - $ fixdecoder --summary --follow logs/fix.log +Obfuscate sensitive FIX fields while decoding. When enabled, values for a predefined set of sensitive tags (e.g., session IDs, sender/target IDs) are replaced with stable aliases (e.g., `SenderCompID0001`) so logs stay readable without exposing real identifiers. Obfuscation is applied per line/message and resets between files; disabled by default. -``` - +### `--colour[=yes|no]` -```bash -❯ target/debug/fixdecoder --info -fixdecoder 0.2.0 (branch:develop, commit:7a2d535) [rust:1.91.1] -Available FIX Dictionaries: FIX27,FIX30,FIX40,FIX41,FIX42,FIX43,FIX44,FIX50,FIX50SP1,FIX50SP2,FIXT11 - -Loaded dictionaries: - Version ServicePack Fields Components Messages Source - FIX27 0 138 2 27 built-in - FIX30 0 138 2 27 built-in - FIX40 0 138 2 27 built-in - FIX41 0 206 2 28 built-in - FIX42 0 405 2 46 built-in - FIX43 0 635 12 68 built-in - FIX44 0 912 106 93 built-in - FIX50 0 1090 123 93 built-in - FIX50SP1 1 1373 165 105 built-in - FIX50SP2 2 6028 727 156 built-in - FIXT11 0 71 4 8 built-in -``` +Control coloured output. By default, colours are shown when writing to a terminal and disabled when output is piped. Use `--colour`/`--colour=yes` to force colours on, or `--colour=no` to force them off. Non-tty output defaults to no colour unless you explicitly opt in. + +### `--delimiter=` + +Set the display delimiter between FIX fields (default: `SOH`). Specify a single character after `=` sign. + +Accepted values: + +- A single literal character (e.g.`,`, `|`, or a single Unicode character like `—`). + +- SOH (case-insensitive) or a hex escape like `\x01`/`0x01` (quote to protect the backslash, e.g. `--delimiter='\x1f'`). + +Empty values or anything longer than one character are rejected. + +### `-f`, `--follow` -## Download it +Stream input like `tail -f`. Keeps reading and decoding as new data arrives on stdin or a file, sleeping briefly on `EOF` rather than exiting, until interrupted. This mirrors `tail -f` behaviour but with FIX decoding, validation, and prettification applied in real time. -Check out the Repo's [Releases Page](https://github.com/stephenlclarke/fixdecoder2/releases) -to see what versions are available for the computer you want to run it on. +### `--summary` -## Build it +Track FIX order lifecycles and emit a summary instead of full decoded messages. When enabled, each message is consumed into an order tracker (keyed by `OrderID`/`ClOrdID`/`OrigClOrdID`), updating state, quantities, prices, and events. At the end (or live in `--follow` mode) it prints a concise per-order summary/footer using the chosen display delimiter. This mode suppresses the usual prettified message output; use it to monitor order state across a stream or log. + +# Download it + +Check out the Repo's [Releases Page](https://github.com/stephenlclarke/fixdecoder2/releases) to see what versions are available for the computer you want to run it on. + +# Build it Build it from source. This now requires `bash` version 5+ and a recent `Rust` toolchain (the project is tested with Rust 1.91+). @@ -197,7 +172,8 @@ Resolving deltas: 100% (201/201), done. Then build it. Debug version with clippy and code coverage ```bash -❯ make build scan coverage +❯ make clean build scan coverage build-release + Removed 51032 files, 3.9GiB total >> Ensuring Rust toolchain and coverage tools @@ -205,92 +181,355 @@ Then build it. Debug version with clippy and code coverage info: component 'llvm-tools' for target 'aarch64-apple-darwin' is up to date >> Ensuring FIX XML specs are present - Compiling fixdecoder v0.2.0 (/Users/sclarke/github/fixdecoder2) -warning: fixdecoder@0.2.0: Building fixdecoder 0.2.0 (branch:develop, commit:d7de4f4) [rust:1.91.1] - Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.92s - Compiling fixdecoder v0.2.0 (/Users/sclarke/github/fixdecoder2) -warning: fixdecoder@0.2.0: Building fixdecoder 0.2.0 (branch:develop, commit:d7de4f4) [rust:1.91.1] - Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.71s - Fetching advisory database from `https://github.com/RustSec/advisory-db.git` + Compiling minimal-lexical v0.2.1 + Compiling thiserror v1.0.69 + Compiling fixdecoder v0.3.0 (/Users/sclarke/github/fixdecoder2) + Compiling thiserror-impl v1.0.69 + Compiling arrayvec v0.7.6 + Compiling circular v0.3.0 + Compiling etherparse v0.15.0 + Compiling nom v7.1.3 +warning: fixdecoder@0.3.0: Building fixdecoder 0.3.0 (branch:develop, commit:9f467f8) [rust:1.91.1] + Compiling rusticata-macros v4.1.0 + Compiling pcap-parser v0.14.1 + Compiling pcap2fix v0.1.0 (/Users/sclarke/github/fixdecoder2/pcap2fix) + Finished `dev` profile [unoptimized + debuginfo] target(s) in 1.88s + Checking memchr v2.7.6 + Checking anstyle v1.0.13 + Checking utf8parse v0.2.2 + Checking bitflags v2.10.0 + Checking regex-syntax v0.8.8 + Checking libc v0.2.178 + Checking anstyle-query v1.1.5 + Checking is_terminal_polyfill v1.70.2 + Checking colorchoice v1.0.4 + Checking crossbeam-utils v0.8.21 + Checking num-traits v0.2.19 + Checking strsim v0.11.1 + Checking anstyle-parse v0.2.7 + Checking cfg-if v1.0.4 + Compiling rustix v1.1.2 + Checking objc2-encode v4.1.0 + Checking clap_lex v0.7.6 + Checking serde_core v1.0.228 + Checking anyhow v1.0.100 + Checking predicates-core v1.0.9 + Checking core-foundation-sys v0.8.7 + Checking objc2 v0.6.3 + Checking anstream v0.6.21 + Compiling fixdecoder v0.3.0 (/Users/sclarke/github/fixdecoder2) + Checking crossbeam-epoch v0.9.18 + Checking aho-corasick v1.1.4 + Checking normalize-line-endings v0.3.0 + Checking clap_builder v4.5.53 + Checking iana-time-zone v0.1.64 + Checking crossbeam-deque v0.8.6 + Checking either v1.15.0 + Checking termtree v0.5.1 + Checking float-cmp v0.10.0 + Checking difflib v0.4.0 + Checking rayon-core v1.13.0 + Checking once_cell v1.21.3 + Compiling assert_cmd v2.1.1 + Compiling getrandom v0.3.4 + Checking predicates-tree v1.0.12 + Checking chrono v0.4.42 + Checking rayon v1.11.0 + Checking errno v0.3.14 + Checking nix v0.30.1 + Checking wait-timeout v0.2.1 + Checking roxmltree v0.21.1 + Checking block2 v0.6.2 +warning: fixdecoder@0.3.0: Building fixdecoder 0.3.0 (branch:develop, commit:9f467f8) [rust:1.91.1] + Checking minimal-lexical v0.2.1 + Checking fastrand v2.3.0 + Checking arrayvec v0.7.6 + Checking dispatch2 v0.3.0 + Checking nom v7.1.3 + Checking circular v0.3.0 + Checking etherparse v0.15.0 + Checking regex-automata v0.4.13 + Checking thiserror v1.0.69 + Checking clap v4.5.53 + Checking ctrlc v3.5.1 + Checking serde v1.0.228 + Checking terminal_size v0.4.3 + Checking quick-xml v0.36.2 + Checking tempfile v3.23.0 + Checking rusticata-macros v4.1.0 + Checking pcap-parser v0.14.1 + Checking pcap2fix v0.1.0 (/Users/sclarke/github/fixdecoder2/pcap2fix) + Checking regex v1.12.2 + Checking bstr v1.12.1 + Checking predicates v3.1.3 + Finished `dev` profile [unoptimized + debuginfo] target(s) in 3.30s +Running cargo-audit (text output) Loaded 884 security advisories (from /Users/sclarke/.cargo/advisory-db) - Updating crates.io index - Scanning Cargo.lock for vulnerabilities (110 crate dependencies) -Crate: atty -Version: 0.2.14 -Warning: unmaintained -Title: `atty` is unmaintained -Date: 2024-09-25 -ID: RUSTSEC-2024-0375 -URL: https://rustsec.org/advisories/RUSTSEC-2024-0375 -Dependency tree: -atty 0.2.14 -└── fixdecoder 0.2.0 - -Crate: atty -Version: 0.2.14 -Warning: unsound -Title: Potential unaligned read -Date: 2021-07-04 -ID: RUSTSEC-2021-0145 -URL: https://rustsec.org/advisories/RUSTSEC-2021-0145 - -warning: 2 allowed warnings found + Scanning Cargo.lock for vulnerabilities (115 crate dependencies) +Running cargo-audit (JSON) → target/coverage/rustsec.json +Converting RustSec report to Sonar generic issues (target/coverage/sonar-generic-issues.json) info: cargo-llvm-cov currently setting cfg(coverage); you can opt-out it by passing --no-cfg-coverage - Compiling fixdecoder v0.2.0 (/Users/sclarke/github/fixdecoder2) -warning: fixdecoder@0.2.0: Building fixdecoder 0.2.0 (branch:develop, commit:d7de4f4) [rust:1.91.1] - Finished `test` profile [unoptimized + debuginfo] target(s) in 1.98s - Running unittests src/main.rs (target/llvm-cov-target/debug/deps/fixdecoder-f3e1ddf320589917) - -running 27 tests + Compiling libc v0.2.178 + Compiling memchr v2.7.6 + Compiling proc-macro2 v1.0.103 + Compiling quote v1.0.42 + Compiling unicode-ident v1.0.22 + Compiling autocfg v1.5.0 + Compiling regex-syntax v0.8.8 + Compiling crossbeam-utils v0.8.21 + Compiling bitflags v2.10.0 + Compiling anstyle v1.0.13 + Compiling objc2 v0.6.3 + Compiling utf8parse v0.2.2 + Compiling anstyle-parse v0.2.7 + Compiling cfg-if v1.0.4 + Compiling colorchoice v1.0.4 + Compiling is_terminal_polyfill v1.70.2 + Compiling rustix v1.1.2 + Compiling cfg_aliases v0.2.1 + Compiling serde_core v1.0.228 + Compiling objc2-encode v4.1.0 + Compiling anstyle-query v1.1.5 + Compiling anstream v0.6.21 + Compiling num-traits v0.2.19 + Compiling aho-corasick v1.1.4 + Compiling nix v0.30.1 + Compiling rayon-core v1.13.0 + Compiling strsim v0.11.1 + Compiling anyhow v1.0.100 + Compiling crossbeam-epoch v0.9.18 + Compiling serde v1.0.228 + Compiling semver v1.0.27 + Compiling heck v0.5.0 + Compiling clap_lex v0.7.6 + Compiling clap_builder v4.5.53 + Compiling regex-automata v0.4.13 + Compiling block2 v0.6.2 + Compiling syn v2.0.111 + Compiling rustc_version v0.4.1 + Compiling crossbeam-deque v0.8.6 + Compiling core-foundation-sys v0.8.7 + Compiling predicates-core v1.0.9 + Compiling iana-time-zone v0.1.64 + Compiling errno v0.3.14 + Compiling regex v1.12.2 + Compiling dispatch2 v0.3.0 + Compiling fixdecoder v0.3.0 (/Users/sclarke/github/fixdecoder2) + Compiling once_cell v1.21.3 + Compiling termtree v0.5.1 + Compiling difflib v0.4.0 + Compiling getrandom v0.3.4 + Compiling assert_cmd v2.1.1 + Compiling either v1.15.0 + Compiling float-cmp v0.10.0 + Compiling normalize-line-endings v0.3.0 + Compiling predicates v3.1.3 + Compiling terminal_size v0.4.3 + Compiling chrono v0.4.42 + Compiling predicates-tree v1.0.12 + Compiling bstr v1.12.1 + Compiling wait-timeout v0.2.1 + Compiling ctrlc v3.5.1 + Compiling roxmltree v0.21.1 + Compiling rayon v1.11.0 + Compiling fastrand v2.3.0 + Compiling minimal-lexical v0.2.1 + Compiling nom v7.1.3 + Compiling thiserror v1.0.69 + Compiling tempfile v3.23.0 + Compiling clap_derive v4.5.49 + Compiling serde_derive v1.0.228 + Compiling thiserror-impl v1.0.69 +warning: fixdecoder@0.3.0: Building fixdecoder 0.3.0 (branch:develop, commit:9f467f8) [rust:1.91.1] + Compiling circular v0.3.0 + Compiling arrayvec v0.7.6 + Compiling etherparse v0.15.0 + Compiling clap v4.5.53 + Compiling rusticata-macros v4.1.0 + Compiling pcap-parser v0.14.1 + Compiling pcap2fix v0.1.0 (/Users/sclarke/github/fixdecoder2/pcap2fix) + Compiling quick-xml v0.36.2 + Finished `test` profile [unoptimized + debuginfo] target(s) in 8.38s + Running unittests src/main.rs (target/llvm-cov-target/debug/deps/fixdecoder-ae50d2dc6f6148f6) + +running 75 tests +test decoder::display::tests::layout_stats_produces_layout ... ok +test decoder::display::tests::pad_ansi_extends_to_requested_width ... ok +test decoder::display::tests::collect_group_layout_counts_nested_components ... ok +test decoder::display::tests::print_field_renders_required_indicator ... ok +test decoder::display::tests::compute_values_layout_uses_max_entry ... ok +test decoder::display::tests::print_enum_outputs_coloured_enum ... ok +test decoder::display::tests::tag_and_message_cells_include_expected_text ... ok +test decoder::display::tests::collect_sorted_values_orders_by_enum ... ok +test decoder::display::tests::terminal_width_is_positive ... ok +test decoder::display::tests::compute_message_layout_counts_header_and_trailer ... ok +test decoder::display::tests::visible_len_ignores_escape_sequences ... ok +test decoder::display::tests::visible_width_ignores_ansi_sequences ... ok +test decoder::display::tests::print_enum_columns_respects_layout_columns ... ok +test decoder::display::tests::write_with_padding_adds_spaces ... ok +test decoder::display::tests::render_component_prints_matching_msg_type_enum_only ... ok +test decoder::display::tests::render_message_includes_header_and_trailer ... ok +test decoder::display::tests::cached_layout_is_reused_for_component ... ok +test decoder::prettifier::tests::trim_line_endings_strips_crlf ... ok +test decoder::prettifier::tests::read_line_with_follow_returns_zero_on_eof ... ok test decoder::schema::tests::parse_simple_vec ... ok test decoder::schema::tests::parse_message_fields ... ok test decoder::schema::tests::parse_message_with_components ... ok -test decoder::tag_lookup::tests::detects_schema_from_default_appl_ver_id ... ok +test decoder::prettifier::tests::prettify_aligns_group_entries_without_header ... ok +test decoder::prettifier::tests::header_and_trailer_are_repositioned_when_out_of_place ... ok +test decoder::prettifier::tests::build_tag_order_respects_annotations_and_trailer ... ok +test decoder::summary::tests::build_summary_row_includes_bn_headers ... ok +test decoder::summary::tests::display_instrument_formats_side_and_symbol ... ok +test decoder::summary::tests::date_diff_days_returns_none_when_incomplete ... ok +test decoder::summary::tests::extract_date_part_handles_timestamp ... ok +test decoder::summary::tests::flow_label_skips_leading_unknown ... ok +test decoder::summary::tests::preferred_settlement_date_prefers_primary_then_secondary ... ok +test decoder::summary::tests::render_record_header_includes_id_and_instrument ... ok +test decoder::summary::tests::resolve_key_prefers_alias_then_ids ... ok +test decoder::summary::tests::state_path_deduplicates_consecutive_states ... ok test decoder::summary::tests::terminal_status_from_non_exec_report_updates_header ... ok -test decoder::summary::tests::links_orders_using_order_id_and_cl_ord_id ... ok +test decoder::tag_lookup::tests::detects_schema_from_default_appl_ver_id ... ok +test decoder::summary::tests::absorb_fields_sets_core_values ... ok test decoder::summary::tests::render_outputs_state_headline ... ok +test decoder::summary::tests::bn_message_sets_state_and_spot_price ... ok +test decoder::summary::tests::absorb_fields_sets_block_notice_specifics ... ok test decoder::validator::tests::allows_repeating_group_tags ... ok test decoder::validator::tests::detects_body_length_mismatch ... ok test decoder::validator::tests::detects_checksum_mismatch ... ok test decoder::validator::tests::missing_msg_type_still_reports_length_and_tag ... ok test fix::obfuscator::tests::reset_starts_aliases_over ... ok +test tests::add_entity_arg_defaults_to_true_when_missing_value ... ok +test tests::add_flag_args_sets_flags ... ok +test tests::build_cli_parses_follow_and_summary_flags ... ok +test tests::dictionary_key_includes_service_pack ... ok +test tests::dictionary_marker_highlights_selected_entry ... ok +test tests::dictionary_source_prefers_custom_entry ... ok +test tests::final_exit_code_marks_interrupt ... ok test tests::invalid_fix_version_errors ... ok +test tests::normalise_fix_key_handles_variants ... ok +test tests::parse_colour_recognises_yes_no ... ok +test tests::parse_colour_rejects_invalid ... ok +test decoder::summary::tests::links_orders_using_order_id_and_cl_ord_id ... ok +test tests::parse_delimiter_accepts_hex ... ok +test tests::parse_delimiter_accepts_literal ... ok +test tests::parse_delimiter_rejects_empty ... ok +test tests::resolve_input_files_defaults_to_stdin ... ok +test tests::resolve_input_files_preserves_inputs ... ok test tests::valid_fix_version_passes ... ok test tests::version_str_is_cached ... ok test tests::version_string_matches_components ... ok -test decoder::prettifier::tests::validation_inserts_missing_tags ... ok -test decoder::prettifier::tests::validation_skips_valid_messages ... ok -test decoder::prettifier::tests::prettify_orders_without_msg_type_header_first ... ok -test decoder::prettifier::tests::validation_only_outputs_invalid_messages ... ok -test decoder::prettifier::tests::header_and_trailer_are_repositioned_when_out_of_place ... ok test decoder::prettifier::tests::prettify_includes_missing_tag_annotations_once ... ok -test decoder::summary::tests::bn_message_sets_state_and_spot_price ... ok +test decoder::prettifier::tests::prettify_orders_without_msg_type_header_first ... ok test decoder::summary::tests::collects_states_for_single_order ... ok +test decoder::prettifier::tests::validation_inserts_missing_tags ... ok +test decoder::prettifier::tests::validation_only_outputs_invalid_messages ... ok +test decoder::prettifier::tests::validation_skips_valid_messages ... ok test decoder::tag_lookup::tests::load_dictionary_respects_override_key ... ok test decoder::tag_lookup::tests::override_uses_fallback_dictionary_for_missing_tags ... ok +test decoder::tag_lookup::tests::repeatable_tags_include_nested_groups ... ok test decoder::tag_lookup::tests::warns_and_falls_back_on_unknown_override ... ok -test result: ok. 27 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 1.01s +test result: ok. 75 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 3.00s - Running unittests src/bin/generate_sensitive_tags.rs (target/llvm-cov-target/debug/deps/generate_sensitive_tags-1dc73cddc48f727b) + Running unittests src/bin/generate_sensitive_tags.rs (target/llvm-cov-target/debug/deps/generate_sensitive_tags-79d8bd38aa9fdc5a) running 0 tests test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - Running tests/cli.rs (target/llvm-cov-target/debug/deps/cli-44905c680ca51135) + Running tests/cli.rs (target/llvm-cov-target/debug/deps/cli-762abd0244a0dac8) running 5 tests +test summary_mode_outputs_order_summary ... ok test decodes_single_message_from_stdin ... ok test validation_reports_missing_fields ... ok -test summary_mode_outputs_order_summary ... ok test decodes_message_from_file_path ... ok test override_is_honoured_with_fallback ... ok -test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.78s +test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 3.24s + + Running unittests src/main.rs (target/llvm-cov-target/debug/deps/pcap2fix-19171fc4d58700e9) + +running 6 tests +test tests::out_of_order_future_segment_is_skipped ... ok +test tests::flushes_full_messages_only ... ok +test tests::flush_complete_messages_emits_and_retains_tail ... ok +test tests::parse_delimiter_variants ... ok +test tests::reassembly_appends_in_order ... ok +test tests::retransmit_is_ignored ... ok + +test result: ok. 6 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s Finished report saved to target/coverage/coverage.xml + Compiling proc-macro2 v1.0.103 + Compiling quote v1.0.42 + Compiling unicode-ident v1.0.22 + Compiling memchr v2.7.6 + Compiling libc v0.2.178 + Compiling crossbeam-utils v0.8.21 + Compiling bitflags v2.10.0 + Compiling objc2 v0.6.3 + Compiling utf8parse v0.2.2 + Compiling colorchoice v1.0.4 + Compiling cfg_aliases v0.2.1 + Compiling is_terminal_polyfill v1.70.2 + Compiling anstyle-parse v0.2.7 + Compiling serde_core v1.0.228 + Compiling autocfg v1.5.0 + Compiling objc2-encode v4.1.0 + Compiling anstyle-query v1.1.5 + Compiling anstyle v1.0.13 + Compiling nix v0.30.1 + Compiling anstream v0.6.21 + Compiling semver v1.0.27 + Compiling num-traits v0.2.19 + Compiling clap_lex v0.7.6 + Compiling rustix v1.1.2 + Compiling serde v1.0.228 + Compiling anyhow v1.0.100 + Compiling strsim v0.11.1 + Compiling rayon-core v1.13.0 + Compiling heck v0.5.0 + Compiling clap_builder v4.5.53 + Compiling block2 v0.6.2 + Compiling crossbeam-epoch v0.9.18 + Compiling crossbeam-deque v0.8.6 + Compiling rustc_version v0.4.1 + Compiling syn v2.0.111 + Compiling aho-corasick v1.1.4 + Compiling minimal-lexical v0.2.1 + Compiling core-foundation-sys v0.8.7 + Compiling cfg-if v1.0.4 + Compiling regex-syntax v0.8.8 + Compiling errno v0.3.14 + Compiling dispatch2 v0.3.0 + Compiling iana-time-zone v0.1.64 + Compiling nom v7.1.3 + Compiling fixdecoder v0.3.0 (/Users/sclarke/github/fixdecoder2) + Compiling thiserror v1.0.69 + Compiling either v1.15.0 + Compiling regex-automata v0.4.13 + Compiling rusticata-macros v4.1.0 + Compiling rayon v1.11.0 + Compiling terminal_size v0.4.3 + Compiling ctrlc v3.5.1 + Compiling roxmltree v0.21.1 + Compiling arrayvec v0.7.6 + Compiling serde_derive v1.0.228 + Compiling clap_derive v4.5.49 + Compiling thiserror-impl v1.0.69 + Compiling once_cell v1.21.3 +warning: fixdecoder@0.3.0: Building fixdecoder 0.3.0 (branch:develop, commit:9f467f8) [rust:1.91.1] + Compiling circular v0.3.0 + Compiling chrono v0.4.42 + Compiling pcap-parser v0.14.1 + Compiling etherparse v0.15.0 + Compiling regex v1.12.2 + Compiling clap v4.5.53 + Compiling quick-xml v0.36.2 + Compiling pcap2fix v0.1.0 (/Users/sclarke/github/fixdecoder2/pcap2fix) + Finished `release` profile [optimized] target(s) in 10.03s ``` Build the release version @@ -317,6 +556,18 @@ fixdecoder 0.2.0 (branch:develop, commit:7a2d535) [rust:1.91.1] git clone git@github.com:stephenlclarke/fixdecoder2.git ``` +# PCAP to FIX filter (`pcap2fix`) + +The workspace includes a helper that reassembles TCP streams from PCAP data and emits FIX messages to stdout so you can pipe them into `fixdecoder`. I have wrapped it in a shell script (`./scripts/capture_and_decode.sh`) to make it easy to run. + +- Build: `cargo build -p pcap2fix` (also built via `make build`). +- Offline: `pcap2fix --input capture.pcap | fixdecoder` +- Live (needs tcpdump/dumpcap): `tcpdump -i eth0 -w - 'tcp port 9876' | pcap2fix --port 9876 | fixdecoder` +- Delimiter defaults to SOH; override with `--delimiter`. +- Flow buffers are capped (size + idle timeout) to avoid runaway memory during long captures. + +![Capture and Decode](docs/capture_and_decode.png) + # Technical Notes on the use of the `--summary` flag - As messages stream by, the decoder builds one “record” per order (keyed by OrderID/ClOrdID/OrigClOrdID). diff --git a/bin/scan.sh b/ci/scan.sh similarity index 100% rename from bin/scan.sh rename to ci/scan.sh diff --git a/ci/update_readme.py b/ci/update_readme.py deleted file mode 100755 index 96da53d..0000000 --- a/ci/update_readme.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python3 -"""Refresh README usage block from `fixdecoder --help` output.""" - -from __future__ import annotations - -import re -import subprocess -import sys -from pathlib import Path - - -def generate_usage() -> str: - """Return the CLI usage text by invoking the built binary.""" - try: - return subprocess.check_output( - ["./target/release/fixdecoder", "--help"], - text=True, - stderr=subprocess.STDOUT, - ) - except (OSError, subprocess.CalledProcessError) as exc: - sys.stderr.write(f"Failed to run fixdecoder --help: {exc}\n") - sys.exit(1) - - -def update_readme(root: Path, usage: str) -> None: - """Replace the usage block in README.md with the provided text.""" - readme = root / "README.md" - text = readme.read_text(encoding="utf-8") - pattern = r".*?" - replacement = f"\n```bash\n{usage}```\n" - new_text = re.sub(pattern, replacement, text, flags=re.S) - if new_text != text: - readme.write_text(new_text, encoding="utf-8") - - -def main() -> int: - """Entry point to refresh the README usage block.""" - root = Path(__file__).resolve().parent.parent - usage = generate_usage() - update_readme(root, usage) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/docs/capture_and_decode.png b/docs/capture_and_decode.png new file mode 100644 index 0000000..e552b5e Binary files /dev/null and b/docs/capture_and_decode.png differ diff --git a/docs/info_command.png b/docs/info_command.png new file mode 100644 index 0000000..2afa932 Binary files /dev/null and b/docs/info_command.png differ diff --git a/pcap2fix/Cargo.toml b/pcap2fix/Cargo.toml new file mode 100644 index 0000000..e1c015b --- /dev/null +++ b/pcap2fix/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "pcap2fix" +version = "0.1.0" +edition = "2021" +license = "AGPL-3.0-only" +description = "PCAP to FIX stream filter: reassembles TCP payloads and emits FIX messages" +repository = "https://github.com/stephenlclarke/fixdecoder2" + +[dependencies] +anyhow = "1.0" +clap = { version = "4.5", features = ["derive"] } +pcap-parser = { version = "0.14", features = ["data"] } +etherparse = "0.15" +thiserror = "1.0" + +[dev-dependencies] +assert_cmd = "2.0" +predicates = "3.1" diff --git a/pcap2fix/src/main.rs b/pcap2fix/src/main.rs new file mode 100644 index 0000000..4e857d7 --- /dev/null +++ b/pcap2fix/src/main.rs @@ -0,0 +1,495 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Minimal PCAP-to-FIX filter: reads PCAP (file or stdin), reassembles TCP +// streams, and emits FIX messages separated by the chosen delimiter. + +use anyhow::{anyhow, Context, Result}; +use clap::Parser; +use etherparse::{NetSlice, SlicedPacket, TransportSlice}; +use pcap_parser::data::{get_packetdata, PacketData, ETHERTYPE_IPV4, ETHERTYPE_IPV6}; +use pcap_parser::pcapng::Block; +use pcap_parser::traits::{PcapNGPacketBlock, PcapReaderIterator}; +use pcap_parser::{create_reader, Linktype, PcapBlockOwned}; +use std::collections::HashMap; +use std::fs::File; +use std::io::{self, Write}; +use std::net::Ipv4Addr; +use std::time::{Duration, Instant}; +use thiserror::Error; + +#[derive(Parser, Debug)] +#[command(author, version, about)] +struct Args { + /// PCAP file path or "-" for stdin + #[arg(short, long, default_value = "-")] + input: String, + /// TCP port filter (optional). If omitted, all ports are considered. + #[arg(short = 'p', long)] + port: Option, + /// Message delimiter. Accepts "SOH", literal char, or hex like \x01. + #[arg(short = 'd', long, default_value = "SOH")] + delimiter: String, + /// Max bytes to buffer per flow before eviction + #[arg(long, default_value = "1048576")] + max_flow_bytes: usize, + /// Idle timeout for flows (seconds) + #[arg(long, default_value = "60")] + idle_timeout: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +struct FlowKey { + src: Ipv4Addr, + dst: Ipv4Addr, + sport: u16, + dport: u16, + // direction handled by seq tracking in FlowState +} + +#[derive(Debug)] +struct FlowState { + next_seq: Option, + buffer: Vec, + last_seen: Instant, +} + +impl Default for FlowState { + fn default() -> Self { + FlowState { + next_seq: None, + buffer: Vec::new(), + last_seen: Instant::now(), + } + } +} + +#[derive(Error, Debug)] +enum ReassemblyError { + #[error("flow exceeded max buffer")] + Overflow, +} + +fn main() -> Result<()> { + let args = Args::parse(); + let delimiter = parse_delimiter(&args.delimiter)?; + let mut reader = open_reader(&args.input)?; + + let mut flows: HashMap = HashMap::new(); + let idle = Duration::from_secs(args.idle_timeout); + let mut stdout = io::BufWriter::new(io::stdout().lock()); + let mut scratch = Vec::new(); + let mut legacy_linktype = None; + let mut idb_linktypes: HashMap = HashMap::new(); + let mut next_if_id: u32 = 0; + + loop { + match reader.next() { + Ok((offset, block)) => { + { + match block { + PcapBlockOwned::LegacyHeader(hdr) => { + legacy_linktype = Some(hdr.network); + } + PcapBlockOwned::Legacy(b) => { + let linktype = legacy_linktype.unwrap_or(Linktype::ETHERNET); + if let Some(packet) = + get_packetdata(b.data, linktype, b.caplen as usize) + { + if let Err(err) = handle_packet_data( + packet, + args.port, + delimiter, + args.max_flow_bytes, + &mut flows, + &mut stdout, + ) { + eprintln!("warn: skipping packet: {err}"); + } + } + } + PcapBlockOwned::NG(block) => match block { + Block::SectionHeader(_) => { + idb_linktypes.clear(); + next_if_id = 0; + } + Block::InterfaceDescription(idb) => { + idb_linktypes.insert(next_if_id, idb.linktype); + next_if_id += 1; + } + Block::EnhancedPacket(epb) => { + if let Some(linktype) = idb_linktypes.get(&epb.if_id) { + if let Some(packet) = get_packetdata( + epb.packet_data(), + *linktype, + epb.caplen as usize, + ) { + if let Err(err) = handle_packet_data( + packet, + args.port, + delimiter, + args.max_flow_bytes, + &mut flows, + &mut stdout, + ) { + eprintln!("warn: skipping packet: {err}"); + } + } + } + } + Block::SimplePacket(spb) => { + if let Some(linktype) = idb_linktypes.get(&0) { + if let Some(packet) = get_packetdata( + spb.packet_data(), + *linktype, + spb.origlen as usize, + ) { + if let Err(err) = handle_packet_data( + packet, + args.port, + delimiter, + args.max_flow_bytes, + &mut flows, + &mut stdout, + ) { + eprintln!("warn: skipping packet: {err}"); + } + } + } + } + _ => {} + }, + } + } + reader.consume(offset); + evict_idle(&mut flows, idle); + } + Err(pcap_parser::PcapError::Eof) => break, + Err(pcap_parser::PcapError::Incomplete) => { + // need more data + reader + .refill() + .map_err(|e| anyhow!("failed to refill reader: {e}"))?; + } + Err(e) => return Err(anyhow!("pcap parse error: {e}")), + } + } + + // flush any trailing message fragments (best effort) + for flow in flows.values_mut() { + flush_complete_messages(&mut flow.buffer, delimiter, &mut scratch, &mut stdout)?; + } + stdout.flush()?; + Ok(()) +} + +fn open_reader(path: &str) -> Result> { + if path == "-" { + let stdin = io::stdin(); + create_reader(65536, stdin).map_err(|e| anyhow!("failed to create reader: {e}")) + } else { + let file = File::open(path).with_context(|| format!("open pcap {path}"))?; + create_reader(65536, file).map_err(|e| anyhow!("failed to create reader: {e}")) + } +} + +fn parse_delimiter(raw: &str) -> Result { + if raw.eq_ignore_ascii_case("SOH") { + return Ok(0x01); + } + if let Some(hex) = raw.strip_prefix("\\x").or_else(|| raw.strip_prefix("0x")) { + let val = + u8::from_str_radix(hex, 16).map_err(|_| anyhow!("invalid hex delimiter: {raw}"))?; + return Ok(val); + } + if raw.len() == 1 { + return Ok(raw.as_bytes()[0]); + } + Err(anyhow!( + "delimiter must be SOH, hex (\\x01), or single byte" + )) +} + +fn handle_packet_data( + packet: PacketData<'_>, + port_filter: Option, + delimiter: u8, + max_flow_bytes: usize, + flows: &mut HashMap, + out: &mut W, +) -> Result<()> { + match packet { + PacketData::L2(data) => { + let sliced = SlicedPacket::from_ethernet(data).map_err(|e| anyhow!("parse: {e:?}"))?; + handle_sliced_packet(sliced, port_filter, delimiter, max_flow_bytes, flows, out) + } + PacketData::L3(ethertype, data) + if ethertype == ETHERTYPE_IPV4 || ethertype == ETHERTYPE_IPV6 => + { + let sliced = SlicedPacket::from_ip(data).map_err(|e| anyhow!("parse: {e:?}"))?; + handle_sliced_packet(sliced, port_filter, delimiter, max_flow_bytes, flows, out) + } + _ => Ok(()), + } +} + +fn handle_sliced_packet( + sliced: SlicedPacket<'_>, + port_filter: Option, + delimiter: u8, + max_flow_bytes: usize, + flows: &mut HashMap, + out: &mut W, +) -> Result<()> { + let (ip, tcp) = match (sliced.net, sliced.transport) { + (Some(NetSlice::Ipv4(ip)), Some(TransportSlice::Tcp(tcp))) => (ip, tcp), + _ => return Ok(()), + }; + if let Some(p) = port_filter { + if tcp.source_port() != p && tcp.destination_port() != p { + return Ok(()); + } + } + + let payload = tcp.payload(); + if payload.is_empty() { + return Ok(()); + } + + let header = ip.header(); + let key = FlowKey { + src: header.source_addr(), + dst: header.destination_addr(), + sport: tcp.source_port(), + dport: tcp.destination_port(), + }; + + let seq = tcp.sequence_number(); + let flow = flows.entry(key).or_default(); + flow.last_seen = Instant::now(); + + reassemble_and_emit(flow, seq, payload, delimiter, max_flow_bytes, out) +} + +fn reassemble_and_emit( + flow: &mut FlowState, + seq: u32, + payload: &[u8], + delimiter: u8, + max_flow_bytes: usize, + out: &mut W, +) -> Result<()> { + let expected = flow.next_seq.unwrap_or(seq); + + if seq == expected { + flow.buffer.extend_from_slice(payload); + flow.next_seq = Some(seq.wrapping_add(payload.len() as u32)); + } else if seq > expected { + // out-of-order future segment: skip for now + return Ok(()); + } else { + // retransmit or overlap + let end = seq.wrapping_add(payload.len() as u32); + if end <= expected { + // fully duplicate + return Ok(()); + } + let overlap = (expected - seq) as usize; + flow.buffer.extend_from_slice(&payload[overlap..]); + flow.next_seq = Some(expected.wrapping_add(payload.len() as u32 - overlap as u32)); + } + + if flow.buffer.len() > max_flow_bytes { + flow.buffer.clear(); + return Err(ReassemblyError::Overflow.into()); + } + + let mut scratch = Vec::new(); + flush_complete_messages(&mut flow.buffer, delimiter, &mut scratch, out)?; + Ok(()) +} + +fn flush_complete_messages( + buffer: &mut Vec, + delimiter: u8, + scratch: &mut Vec, + out: &mut W, +) -> Result<()> { + let mut cursor = 0; + while let Some(rel_end) = find_message_end(&buffer[cursor..], delimiter) { + let end = cursor + rel_end; + scratch.clear(); + scratch.extend_from_slice(&buffer[cursor..=end]); + scratch.push(b'\n'); // newline so each FIX message prints on its own line + out.write_all(scratch)?; + cursor = end + 1; + } + if cursor > 0 { + buffer.drain(0..cursor); + } + Ok(()) +} + +fn find_message_end(buffer: &[u8], delimiter: u8) -> Option { + // Need at least "8=..|9=..|" plus checksum ("10=000|") + if buffer.len() < 16 { + return None; + } + let begin_end = buffer.iter().position(|b| *b == delimiter)?; + let body_len_field_start = begin_end + 1; + let body_len_end = body_len_field_start + + buffer[body_len_field_start..] + .iter() + .position(|b| *b == delimiter)?; // include delimiter + if body_len_end <= body_len_field_start + 1 { + return None; + } + if !buffer[body_len_field_start..].starts_with(b"9=") { + return None; + } + let body_len_bytes = &buffer[body_len_field_start + 2..body_len_end]; + let body_len: usize = parse_decimal(body_len_bytes)?; + let body_start = body_len_end + 1; + let body_end = body_start.checked_add(body_len)?; + // checksum starts immediately after body + if body_end + 7 > buffer.len() { + return None; + } + if !buffer.get(body_end..)?.starts_with(b"10=") { + return None; + } + let checksum_val = buffer.get(body_end + 3..body_end + 6)?; + if checksum_val.iter().any(|b| !b.is_ascii_digit()) { + return None; + } + let end_delim_idx = body_end + 6; + if *buffer.get(end_delim_idx)? != delimiter { + return None; + } + Some(end_delim_idx) +} + +fn parse_decimal(bytes: &[u8]) -> Option { + let mut val: usize = 0; + for b in bytes { + if !b.is_ascii_digit() { + return None; + } + val = val.checked_mul(10)?; + val = val.checked_add((b - b'0') as usize)?; + } + Some(val) +} +fn evict_idle(flows: &mut HashMap, idle: Duration) { + let now = Instant::now(); + flows.retain(|_, state| now.duration_since(state.last_seen) < idle); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn build_fix_message(body: &str, delim: u8) -> Vec { + let mut msg = Vec::new(); + let d = delim as char; + let body_len = body.len(); + msg.extend_from_slice(format!("8=FIX.4.4{d}9={body_len}{d}").as_bytes()); + msg.extend_from_slice(body.as_bytes()); + let checksum: u8 = msg.iter().fold(0u16, |acc, b| acc + *b as u16) as u8; + msg.extend_from_slice(format!("10={:03}{}", checksum, d).as_bytes()); + msg + } + + #[test] + fn parse_delimiter_variants() { + assert_eq!(parse_delimiter("SOH").unwrap(), 0x01); + assert_eq!(parse_delimiter("\\x02").unwrap(), 0x02); + assert_eq!(parse_delimiter("0x03").unwrap(), 0x03); + assert_eq!(parse_delimiter("|").unwrap(), b'|'); + } + + #[test] + fn reassembly_appends_in_order() { + let mut flow = FlowState::default(); + let mut out = Vec::new(); + let message = build_fix_message("35=0\u{0001}", 0x01); + let (part1, rest) = message.split_at(10); + let (part2, part3) = rest.split_at(8); + + reassemble_and_emit(&mut flow, 10, part1, 0x01, 1024, &mut out).unwrap(); + reassemble_and_emit( + &mut flow, + 10 + part1.len() as u32, + part2, + 0x01, + 1024, + &mut out, + ) + .unwrap(); + assert!(out.is_empty(), "no complete message yet"); + reassemble_and_emit( + &mut flow, + 10 + (part1.len() + part2.len()) as u32, + part3, + 0x01, + 1024, + &mut out, + ) + .unwrap(); + let text = String::from_utf8(out).unwrap(); + assert!(text.contains("8=FIX.4.4")); + assert!(text.ends_with('\n')); + } + + #[test] + fn flushes_full_messages_only() { + let mut buf = build_fix_message("35=0\u{0001}", 0x01); + buf.extend_from_slice(b"extra"); + let mut out = Vec::new(); + let mut scratch = Vec::new(); + flush_complete_messages(&mut buf, 0x01, &mut scratch, &mut out).unwrap(); + let mut expected = build_fix_message("35=0\u{0001}", 0x01); + expected.push(b'\n'); + assert_eq!(out, expected); + assert_eq!(buf.as_slice(), b"extra"); + } + + #[test] + fn retransmit_is_ignored() { + let mut flow = FlowState::default(); + let mut out = Vec::new(); + reassemble_and_emit(&mut flow, 1, b"ABC", b'|', 1024, &mut out).unwrap(); + reassemble_and_emit(&mut flow, 1, b"ABC", b'|', 1024, &mut out).unwrap(); + assert!(flow.buffer.starts_with(b"ABC")); + } + + #[test] + fn out_of_order_future_segment_is_skipped() { + let mut flow = FlowState::default(); + let mut out = Vec::new(); + reassemble_and_emit(&mut flow, 5, b"first", b'|', 1024, &mut out).unwrap(); + // future seq skipped + reassemble_and_emit(&mut flow, 20, b"second", b'|', 1024, &mut out).unwrap(); + assert_eq!(flow.buffer, b"first"); + } + + #[test] + fn flush_complete_messages_emits_and_retains_tail() { + let mut buf = Vec::new(); + let msg1 = build_fix_message("35=0|", b'|'); + let msg2 = build_fix_message("35=1|", b'|'); + buf.extend_from_slice(&msg1); + buf.extend_from_slice(&msg2); + buf.extend_from_slice(b"partial"); + let mut scratch = Vec::new(); + let mut out = Vec::new(); + flush_complete_messages(&mut buf, b'|', &mut scratch, &mut out).unwrap(); + let expected_out = { + let mut v = msg1.clone(); + v.push(b'\n'); + v.extend_from_slice(&msg2); + v.push(b'\n'); + v + }; + assert_eq!(out, expected_out); + assert_eq!(buf, b"partial"); + } +} diff --git a/pcap2fix/tests/roundtrip.rs b/pcap2fix/tests/roundtrip.rs new file mode 100644 index 0000000..0059a5f --- /dev/null +++ b/pcap2fix/tests/roundtrip.rs @@ -0,0 +1,89 @@ +use assert_cmd::Command; + +/// Build a minimal FIX message with correct BodyLength/Checksum using the given delimiter. +fn build_fix_message(delim: u8) -> Vec { + let d = delim as char; + let body = format!("35=0{d}"); + let body_len = body.len(); + let mut msg = format!("8=FIX.4.2{d}9={body_len}{d}{body}").into_bytes(); + let checksum: u8 = msg.iter().fold(0u16, |acc, b| acc + *b as u16) as u8; + msg.extend_from_slice(format!("10={:03}{}", checksum, d).as_bytes()); + msg +} + +/// Construct a tiny PCAP (classic) containing one Ethernet/IPv4/TCP packet with the FIX payload. +fn build_pcap(payload: &[u8]) -> Vec { + let mut buf = Vec::new(); + + // PCAP global header (little-endian, Ethernet linktype) + buf.extend_from_slice(&0xa1b2c3d4u32.to_le_bytes()); // magic + buf.extend_from_slice(&0x0002u16.to_le_bytes()); // version major + buf.extend_from_slice(&0x0004u16.to_le_bytes()); // version minor + buf.extend_from_slice(&0u32.to_le_bytes()); // thiszone + buf.extend_from_slice(&0u32.to_le_bytes()); // sigfigs + buf.extend_from_slice(&65535u32.to_le_bytes()); // snaplen + buf.extend_from_slice(&1u32.to_le_bytes()); // network = Ethernet + + // Build packet bytes + let mut pkt = Vec::new(); + // Ethernet + pkt.extend_from_slice(&[0, 1, 2, 3, 4, 5]); // dst + pkt.extend_from_slice(&[6, 7, 8, 9, 10, 11]); // src + pkt.extend_from_slice(&[0x08, 0x00]); // ethertype IPv4 + // IPv4 header + let ip_header_len = 20u16; + let tcp_header_len = 20u16; + let total_len = ip_header_len + tcp_header_len + payload.len() as u16; + pkt.extend_from_slice(&[0x45, 0x00]); // version/IHL, DSCP + pkt.extend_from_slice(&total_len.to_be_bytes()); + pkt.extend_from_slice(&[0x00, 0x00]); // identification + pkt.extend_from_slice(&[0x40, 0x00]); // flags/frag offset + pkt.extend_from_slice(&[64]); // TTL + pkt.extend_from_slice(&[6]); // protocol TCP + pkt.extend_from_slice(&[0x00, 0x00]); // checksum (omitted) + pkt.extend_from_slice(&[10, 0, 0, 1]); // src IP + pkt.extend_from_slice(&[10, 0, 0, 2]); // dst IP + // TCP header + let src_port: u16 = 40000; + let dst_port: u16 = 12083; + pkt.extend_from_slice(&src_port.to_be_bytes()); + pkt.extend_from_slice(&dst_port.to_be_bytes()); + pkt.extend_from_slice(&1u32.to_be_bytes()); // seq + pkt.extend_from_slice(&0u32.to_be_bytes()); // ack + pkt.extend_from_slice(&[0x50, 0x18]); // data offset=5, flags=PSH+ACK + pkt.extend_from_slice(&0xffffu16.to_be_bytes()); // window + pkt.extend_from_slice(&[0x00, 0x00]); // checksum (omitted) + pkt.extend_from_slice(&[0x00, 0x00]); // urgent ptr + // Payload + pkt.extend_from_slice(payload); + + // PCAP packet header + let pkt_len = pkt.len() as u32; + buf.extend_from_slice(&0u32.to_le_bytes()); // ts_sec + buf.extend_from_slice(&0u32.to_le_bytes()); // ts_usec + buf.extend_from_slice(&pkt_len.to_le_bytes()); // incl_len + buf.extend_from_slice(&pkt_len.to_le_bytes()); // orig_len + + buf.extend_from_slice(&pkt); + buf +} + +#[test] +fn pcap_roundtrip_matches_expected_output() { + let delim = 0x01; + let msg = build_fix_message(delim); + let pcap_bytes = build_pcap(&msg); + let expected_output = { + let mut v = msg.clone(); + v.push(b'\n'); + v + }; + + let bin = assert_cmd::cargo::cargo_bin!("pcap2fix"); + Command::new(bin) + .args(["--input", "-", "--port", "12083"]) + .write_stdin(pcap_bytes) + .assert() + .success() + .stdout(expected_output); +} diff --git a/scripts/capture_and_decode.sh b/scripts/capture_and_decode.sh new file mode 100755 index 0000000..71140c2 --- /dev/null +++ b/scripts/capture_and_decode.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'USAGE' +Usage: capture_and_decode.sh + +Example: + ./scripts/capture_and_decode.sh user@integration.example.com 192.168.1.10 1234 + +Notes: + - is used in both the tcpdump filter and the pcap2fix --port argument. + - Assumes fixdecoder and pcap2fix binaries are available at ./target/release/. +USAGE +} + +if [[ $# -lt 3 ]]; then + usage + exit 1 +fi + +SSH_TARGET="$1" +TCP_HOST="$2" +PORT="$3" +shift 3 +FIXDECODER_ARGS=("$@") + +REMOTE_CMD="sudo tcpdump -U -n -s0 -i any -w - \"(host ${TCP_HOST} and port ${PORT}) and tcp[((tcp[12] & 0xf0) >> 2):4] = 0x383d4649 and tcp[((tcp[12] & 0xf0) >> 2) + 4] = 0x58\"" + +# Resolve binaries: prefer explicit env override, then PATH, then local release build. +resolve_bin() { + local env_path="$1" + local name="$2" + local local_fallback="$3" + + if [[ -n "${env_path}" ]]; then + echo "${env_path}" + return + fi + + if command -v "${name}" >/dev/null 2>&1; then + command -v "${name}" + return + fi + + echo "${local_fallback}" +} + +PCAP2FIX_BIN="$(resolve_bin "${PCAP2FIX_BIN:-}" pcap2fix ./target/release/pcap2fix)" +FIXDECODER_BIN="$(resolve_bin "${FIXDECODER_BIN:-}" fixdecoder ./target/release/fixdecoder)" + +if [[ ! -x "${PCAP2FIX_BIN}" || ! -x "${FIXDECODER_BIN}" ]]; then + echo "error: expected binaries at ${PCAP2FIX_BIN} and ${FIXDECODER_BIN}. Build them first (cargo build --release)." >&2 + exit 1 +fi + +ssh "${SSH_TARGET}" "${REMOTE_CMD}" \ + | "${PCAP2FIX_BIN}" --port "${PORT}" \ + | "${FIXDECODER_BIN}" --follow "${FIXDECODER_ARGS[@]}" diff --git a/bin/slowcat.sh b/scripts/slowcat.sh similarity index 94% rename from bin/slowcat.sh rename to scripts/slowcat.sh index 9c7b66a..69218d4 100755 --- a/bin/slowcat.sh +++ b/scripts/slowcat.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -delay=10 # default sleep delay +delay=1s # default sleep delay # Parse options while getopts "d:" opt; do diff --git a/src/decoder/display.rs b/src/decoder/display.rs index d26c2ed..482b7f8 100644 --- a/src/decoder/display.rs +++ b/src/decoder/display.rs @@ -7,6 +7,7 @@ //! The module-level comment keeps the tone informal yet informative. use crate::decoder::colours::{ColourPalette, palette}; +use crate::decoder::layout::{NEST_INDENT, TAG_WIDTH}; use crate::decoder::schema::{ ComponentNode, Field, FieldNode, GroupNode, MessageNode, SchemaTree, Value, }; @@ -148,7 +149,7 @@ pub(crate) fn pad_ansi(text: &str, width: usize) -> String { /// Tiny helper that implements `Display` for indentation without building /// temporary `String`s. #[derive(Clone, Copy)] -struct Indent(usize); +pub(crate) struct Indent(usize); impl fmt::Display for Indent { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -156,7 +157,7 @@ impl fmt::Display for Indent { } } -fn indent(level: usize) -> Indent { +pub(crate) fn indent(level: usize) -> Indent { Indent(level) } @@ -274,6 +275,20 @@ mod tests { }); let aux_field = sample_field_node(false); + let group_count_field = Arc::new(Field { + name: "Nested".into(), + number: 200, + field_type: "NUMINGROUP".into(), + values: Vec::new(), + values_wrapper: ValuesWrapper::default(), + }); + let allocs_count_field = Arc::new(Field { + name: "Allocs".into(), + number: 201, + field_type: "NUMINGROUP".into(), + values: Vec::new(), + values_wrapper: ValuesWrapper::default(), + }); let group_field = sample_field_node(true); let component = ComponentNode { @@ -330,6 +345,8 @@ mod tests { let mut fields = BTreeMap::new(); fields.insert(msg_type_field.name.clone(), msg_type_field.clone()); fields.insert(aux_field.field.name.clone(), aux_field.field.clone()); + fields.insert(group_count_field.name.clone(), group_count_field.clone()); + fields.insert(allocs_count_field.name.clone(), allocs_count_field.clone()); let mut components = BTreeMap::new(); components.insert(header.name.clone(), header); @@ -410,8 +427,8 @@ mod tests { assert!(s.contains("Header")); assert!(s.contains("Message: ")); assert!(s.contains("Body")); - assert!(s.contains("Group:")); assert!(s.contains("Allocs")); + assert!(s.contains("201")); // group count tag number assert!(s.contains("Trailer")); } @@ -513,7 +530,7 @@ fn print_field( ) -> io::Result<()> { writeln!( out, - "{}{}{:4}{}: {}{}{} ({}{}{}){}", + "{}{}{:width$}{}: {}{}{} ({}{}{}){}", indent(indent_level), colours.tag, field.field.number, @@ -524,7 +541,8 @@ fn print_field( colours.value, field.field.field_type, colours.reset, - format_required(field.required, colours) + format_required(field.required, colours), + width = TAG_WIDTH ) } @@ -829,12 +847,17 @@ impl<'a, 'b, W: Write> RenderContext<'a, 'b, W> { colours.reset )?; - self.print_field_collection(&msg.fields, indent_level, shared_style)?; + self.print_field_collection(&msg.fields, indent_level + 2, shared_style)?; for component in &msg.components { - self.render_component_with_style(Some(msg), component, indent_level, shared_style)?; + self.render_component_with_style( + Some(msg), + component, + indent_level + NEST_INDENT, + shared_style, + )?; } for group in &msg.groups { - self.render_group_with_style(group, indent_level, shared_style)?; + self.render_group_with_style(group, indent_level + NEST_INDENT, shared_style)?; } if include_trailer && let Some(trailer) = self.schema.components.get("Trailer") { @@ -877,18 +900,18 @@ impl<'a, 'b, W: Write> RenderContext<'a, 'b, W> { )?; for field in &component.fields { - print_field(self.out, field, indent_level + 4, colours)?; + print_field(self.out, field, indent_level + NEST_INDENT, colours)?; if self.verbose { - self.print_enums_for_field(field, msg, indent_level + 6, style)?; + self.print_enums_for_field(field, msg, indent_level + NEST_INDENT + 2, style)?; } } for sub in &component.components { - self.render_component_with_style(msg, sub, indent_level + 4, style)?; + self.render_component_with_style(msg, sub, indent_level + NEST_INDENT, style)?; } for group in &component.groups { - self.render_group_with_style(group, indent_level + 4, style)?; + self.render_group_with_style(group, indent_level + NEST_INDENT, style)?; } Ok(()) } @@ -911,24 +934,32 @@ impl<'a, 'b, W: Write> RenderContext<'a, 'b, W> { style }; let colours = style.colours(); - writeln!( - self.out, - "{}Group: {}{}{}{}", - indent(indent_level), - colours.name, - group.name, - colours.reset, - format_required(group.required, colours) - )?; + if let Some(count_field) = self.schema.fields.get(&group.name) { + let count_node = FieldNode { + required: group.required, + field: count_field.clone(), + }; + print_field(self.out, &count_node, indent_level, colours)?; + } else { + writeln!( + self.out, + "{}Group: {}{}{}{}", + indent(indent_level), + colours.name, + group.name, + colours.reset, + format_required(group.required, colours) + )?; + } - self.print_field_collection(&group.fields, indent_level + 4, style)?; + self.print_field_collection(&group.fields, indent_level + NEST_INDENT, style)?; for component in &group.components { - self.render_component_with_style(None, component, indent_level + 4, style)?; + self.render_component_with_style(None, component, indent_level + NEST_INDENT, style)?; } for sub_group in &group.groups { - self.render_group_with_style(sub_group, indent_level + 4, style)?; + self.render_group_with_style(sub_group, indent_level + NEST_INDENT, style)?; } Ok(()) } @@ -1265,22 +1296,22 @@ fn collect_component_layout( indent_level: usize, stats: &mut LayoutStats, ) { - collect_fields_layout(&component.fields, indent_level + 6, stats); + collect_fields_layout(&component.fields, indent_level + NEST_INDENT + 2, stats); for sub in &component.components { - collect_component_layout(sub, indent_level + 4, stats); + collect_component_layout(sub, indent_level + NEST_INDENT, stats); } for group in &component.groups { - collect_group_layout(group, indent_level + 4, stats); + collect_group_layout(group, indent_level + NEST_INDENT, stats); } } fn collect_group_layout(group: &GroupNode, indent_level: usize, stats: &mut LayoutStats) { - collect_fields_layout(&group.fields, indent_level + 6, stats); + collect_fields_layout(&group.fields, indent_level + NEST_INDENT + 2, stats); for component in &group.components { - collect_component_layout(component, indent_level + 4, stats); + collect_component_layout(component, indent_level + NEST_INDENT, stats); } for sub in &group.groups { - collect_group_layout(sub, indent_level + 4, stats); + collect_group_layout(sub, indent_level + NEST_INDENT, stats); } } diff --git a/src/decoder/layout.rs b/src/decoder/layout.rs new file mode 100644 index 0000000..d281316 --- /dev/null +++ b/src/decoder/layout.rs @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2025 Steve Clarke - https://xyzzy.tools +// +// Shared layout constants for FIX rendering across prettifier and schema display. + +/// Width used when printing tag numbers (right-aligned). +pub const TAG_WIDTH: usize = 4; +/// Base indent applied to top-level prettifier fields. +pub const BASE_INDENT: usize = 2; +/// Indent increment for nested components/groups. +pub const NEST_INDENT: usize = 4; +/// Column offset to align group separators under the first parenthesis of the field name. +pub const NAME_TEXT_OFFSET: usize = TAG_WIDTH + 1; +/// Indent applied to entries inside a repeating group (relative to the group's own indent). +pub const ENTRY_FIELD_INDENT: usize = TAG_WIDTH + 1; diff --git a/src/decoder/mod.rs b/src/decoder/mod.rs index 43f3bb1..9d4fd7f 100644 --- a/src/decoder/mod.rs +++ b/src/decoder/mod.rs @@ -4,6 +4,7 @@ pub mod colours; pub mod display; pub mod fixparser; +pub mod layout; pub mod prettifier; pub mod schema; pub mod summary; diff --git a/src/decoder/prettifier.rs b/src/decoder/prettifier.rs index d6d2f8a..8dc32e3 100644 --- a/src/decoder/prettifier.rs +++ b/src/decoder/prettifier.rs @@ -2,12 +2,16 @@ // SPDX-FileCopyrightText: 2025 Steve Clarke - https://xyzzy.tools use crate::decoder::colours::{disable_colours, palette}; -use crate::decoder::display::{pad_ansi, terminal_width, visible_width}; +use crate::decoder::display::{indent, pad_ansi, terminal_width, visible_width}; use crate::decoder::fixparser::{FieldValue, parse_fix}; +use crate::decoder::layout::{BASE_INDENT, ENTRY_FIELD_INDENT, NAME_TEXT_OFFSET}; use crate::decoder::summary::OrderSummary; #[cfg(test)] use crate::decoder::tag_lookup::MessageDef; -use crate::decoder::tag_lookup::{FixTagLookup, load_dictionary_with_override}; +use crate::decoder::tag_lookup::{ + FixTagLookup, GroupSpec as MessageDefGroupSpec, MessageDef as LookupMessageDef, + load_dictionary_with_override, +}; use crate::decoder::validator; use crate::fix; use once_cell::sync::Lazy; @@ -65,33 +69,194 @@ pub fn prettify_with_report( let fields = parse_fix(msg); let annotations = report.map(|r| &r.tag_errors); - let mut tag_buckets = bucket_fields(&fields); - let ordered_tags = build_tag_order(&fields, dict, annotations); + let mut seen_tags = HashSet::new(); + let msg_def = fields + .iter() + .find(|f| f.tag == 35) + .and_then(|f| dict.message_def(&f.value)); + let renderer = msg_def.map(|def| GroupRenderer { + dict, + annotations, + colours: &colours, + msg_def: def, + fields: &fields, + }); - for tag in ordered_tags { - if let Some(bucket) = tag_buckets.get_mut(&tag) { - while let Some(field) = bucket.pop_front() { - write_field_line(&mut output, dict, field, annotations, &colours); - } - } else if let Some(errs) = annotations - .and_then(|ann| ann.get(&tag)) - .filter(|errs| !errs.is_empty()) + let mut idx = 0; + while idx < fields.len() { + let field = &fields[idx]; + seen_tags.insert(field.tag); + if let Some(render) = renderer.as_ref() + && let Some(spec) = render.msg_def.groups.get(&field.tag) { - write_missing_line(&mut output, dict, tag, errs, &colours); + let consumed = render.render_group(&mut output, idx, spec, BASE_INDENT); + idx += consumed.max(1); + } else { + write_field_line(&mut output, dict, field, annotations, &colours, BASE_INDENT); + idx += 1; } } - // Emit any remaining fields that were not covered by ordered_tags. - for bucket in tag_buckets.values_mut() { - while let Some(field) = bucket.pop_front() { - write_field_line(&mut output, dict, field, annotations, &colours); + if let Some(ann) = annotations { + for (tag, errs) in ann { + if seen_tags.contains(tag) || errs.is_empty() { + continue; + } + write_missing_line(&mut output, dict, *tag, errs, &colours); } } output } +struct GroupRenderer<'a> { + dict: &'a FixTagLookup, + annotations: Option<&'a std::collections::HashMap>>, + colours: &'a crate::decoder::colours::ColourPalette, + msg_def: &'a LookupMessageDef, + fields: &'a [FieldValue], +} + +impl<'a> GroupRenderer<'a> { + fn write_field(&self, output: &mut String, field: &FieldValue, indent_spaces: usize) { + write_field_line( + output, + self.dict, + field, + self.annotations, + self.colours, + indent_spaces, + ); + } + + fn render_group( + &self, + output: &mut String, + start_idx: usize, + spec: &MessageDefGroupSpec, + indent_spaces: usize, + ) -> usize { + let mut consumed = 0usize; + let mut entries = 0usize; + let expected = self.fields[start_idx] + .value + .parse::() + .unwrap_or_default(); + self.write_field(output, &self.fields[start_idx], indent_spaces); + let mut idx = start_idx + 1; + while idx < self.fields.len() && entries < expected { + if self.fields[idx].tag != spec.delim { + if self.msg_def.group_membership.get(&self.fields[idx].tag) == Some(&spec.count_tag) + { + if entries == 0 { + let entry_consumed = + self.render_group_entry(output, idx, spec, indent_spaces, entries + 1); + idx += entry_consumed; + entries += 1; + consumed = idx - start_idx; + continue; + } + self.write_field( + output, + &self.fields[idx], + indent_spaces + ENTRY_FIELD_INDENT, + ); + idx += 1; + consumed = idx - start_idx; + continue; + } + break; + } + let entry_consumed = + self.render_group_entry(output, idx, spec, indent_spaces, entries + 1); + idx += entry_consumed; + entries += 1; + consumed = idx - start_idx; + } + + if entries != expected { + if let Some(errs) = self + .annotations + .and_then(|ann| ann.get(&spec.count_tag)) + .filter(|errs| !errs.is_empty()) + { + write_missing_line(output, self.dict, spec.count_tag, errs, self.colours); + } else { + output.push_str(&format!( + "{}{}Warning:{} NumInGroup {} ({}) declared {}, found {}\n", + indent(indent_spaces + 2), + self.colours.error, + self.colours.reset, + spec.count_tag, + spec.name, + expected, + entries + )); + } + } + consumed + } + + fn render_group_entry( + &self, + output: &mut String, + start_idx: usize, + spec: &MessageDefGroupSpec, + indent_spaces: usize, + entry_idx: usize, + ) -> usize { + let entry_label = format!("Group {}", entry_idx); + let dash_count = 60usize.saturating_sub(entry_label.len()); + let dashes = "-".repeat(dash_count); + let dash_start_col = indent_spaces + NAME_TEXT_OFFSET; + let label_indent = dash_start_col.saturating_sub(entry_label.len()); + output.push_str(&format!( + "{}{} {}{}{}\n", + indent(label_indent), + entry_label, + self.colours.error, + dashes, + self.colours.reset + )); + let mut idx = start_idx; + let mut last_pos = -1isize; + while idx < self.fields.len() { + let tag = self.fields[idx].tag; + if tag == spec.delim && idx != start_idx { + break; + } + if let Some(nested) = spec.nested.get(&tag) { + let nested_consumed = + self.render_group(output, idx, nested, indent_spaces + ENTRY_FIELD_INDENT); + idx += nested_consumed.max(1); + continue; + } + if let Some(pos) = spec.entry_pos.get(&tag).copied() { + if (pos as isize) < last_pos + && let Some(errs) = self + .annotations + .and_then(|ann| ann.get(&tag)) + .filter(|errs| !errs.is_empty()) + { + write_missing_line(output, self.dict, tag, errs, self.colours); + } + last_pos = pos as isize; + self.write_field( + output, + &self.fields[idx], + indent_spaces + ENTRY_FIELD_INDENT, + ); + idx += 1; + } else { + break; + } + } + idx - start_idx + } +} + /// Bucket each field by tag so repeat occurrences can be emitted in order. +#[allow(dead_code)] fn bucket_fields( fields: &[FieldValue], ) -> std::collections::HashMap> { @@ -106,6 +271,7 @@ fn bucket_fields( /// Build the emission order of tags using the message definition when known, falling back /// to a header-first order when MsgType is absent, and appending tags referenced in /// validation annotations. +#[allow(dead_code)] fn build_tag_order( fields: &[FieldValue], dict: &FixTagLookup, @@ -144,10 +310,12 @@ fn build_tag_order( final_order } +#[allow(dead_code)] fn canonical_header_tags() -> &'static [u32; 7] { &[8u32, 9, 35, 49, 56, 34, 52] } +#[allow(dead_code)] fn trailer_tags(dict: &FixTagLookup) -> Vec { let order = dict.trailer_tags(); if order.is_empty() { @@ -157,6 +325,7 @@ fn trailer_tags(dict: &FixTagLookup) -> Vec { } } +#[allow(dead_code)] fn collect_trailer_tags(fields: &[FieldValue], trailer_set: &HashSet) -> HashSet { fields .iter() @@ -173,6 +342,7 @@ fn message_field_order(fields: &[FieldValue], dict: &FixTagLookup) -> Option Vec { let mut base = vec![8, 9, 35]; for f in fields { @@ -183,11 +353,13 @@ fn fallback_field_order(fields: &[FieldValue]) -> Vec { base } +#[allow(dead_code)] fn dedup_order(order: Vec) -> Vec { let mut seen = HashSet::new(); order.into_iter().filter(|tag| seen.insert(*tag)).collect() } +#[allow(dead_code)] fn base_message_order( fields: &[FieldValue], dict: &FixTagLookup, @@ -207,6 +379,7 @@ fn base_message_order( deduped } +#[allow(dead_code)] fn append_annotation_tags( final_order: &mut Vec, annotations: &std::collections::HashMap>, @@ -228,6 +401,7 @@ fn append_annotation_tags( } } +#[allow(dead_code)] fn append_message_fields( fields: &[FieldValue], final_order: &mut Vec, @@ -246,6 +420,7 @@ fn append_message_fields( } } +#[allow(dead_code)] fn append_trailer_tags( final_order: &mut Vec, trailer_order: &[u32], @@ -329,6 +504,7 @@ fn write_field_line( field: &crate::decoder::fixparser::FieldValue, annotations: Option<&std::collections::HashMap>>, colours: &crate::decoder::colours::ColourPalette, + indent_spaces: usize, ) { let tag_errors: Option<&Vec> = annotations.and_then(|ann| ann.get(&field.tag)); let tag_colour = if tag_errors.is_some() { @@ -346,7 +522,8 @@ fn write_field_line( let name_section = format!("{}({}){}", colours.name, name_coloured, colours.reset); let desc = dict.enum_description(field.tag, &field.value); output.push_str(&format!( - " {}{:4}{} {}: {}{}{}", + "{}{}{:4}{} {}: {}{}{}", + indent(indent_spaces), tag_colour, field.tag, colours.reset, @@ -386,7 +563,8 @@ fn write_missing_line( errors.join(", ") }; output.push_str(&format!( - " {}{:4}{} ({}{}{}): {}{}{}\n", + "{}{}{:4}{} ({}{}{}): {}{}{}\n", + indent(BASE_INDENT), colours.error, tag, colours.reset, @@ -797,6 +975,8 @@ fn test_lookup_with_order(field_order: Vec) -> FixTagLookup { _msg_type: "X".to_string(), field_order, required: Vec::new(), + groups: HashMap::new(), + group_membership: HashMap::new(), }, ); FixTagLookup::new_for_tests(messages) @@ -805,6 +985,7 @@ fn test_lookup_with_order(field_order: Vec) -> FixTagLookup { #[cfg(test)] mod tests { use super::*; + use crate::decoder::schema::FixDictionary; use crate::decoder::tag_lookup::load_dictionary; use crate::decoder::validator; use crate::fix; @@ -816,6 +997,74 @@ mod tests { static TEST_GUARD: once_cell::sync::Lazy> = once_cell::sync::Lazy::new(|| Mutex::new(())); + fn small_group_lookup() -> FixTagLookup { + let xml = r#" + +
+ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + +"#; + let dict = FixDictionary::from_xml(xml).expect("tiny dictionary parses"); + FixTagLookup::from_dictionary(&dict, "TEST") + } + + #[test] + fn prettify_aligns_group_entries_without_header() { + let _lock = TEST_GUARD.lock().unwrap(); + disable_output_colours(); + let dict = small_group_lookup(); + let msg = format!( + "8=FIX.4.4{SOH}35=W{SOH}268=2{SOH}269=0{SOH}270=12.34{SOH}269=1{SOH}270=56.78{SOH}10=000{SOH}" + ); + let rendered = prettify_with_report(&msg, &dict, None); + assert!( + !rendered.contains("Group: NoMDEntries"), + "group header line should be omitted: {rendered}" + ); + let count_line = rendered + .lines() + .find(|l| l.contains("NoMDEntries")) + .expect("count tag line present"); + let group_line = rendered + .lines() + .find(|l| l.contains("Group 1")) + .expect("group entry label present"); + let paren_col = count_line.find('(').expect("open paren present"); + let dash_col = group_line.find('-').expect("dashes present"); + assert_eq!( + dash_col, + paren_col + 1, + "group separator should start one space after '(' anchor" + ); + } + #[test] fn validation_only_outputs_invalid_messages() { let _lock = TEST_GUARD.lock().unwrap(); @@ -989,6 +1238,8 @@ mod tests { _msg_type: "X".to_string(), field_order: vec![8, 9, 35, 55], required: Vec::new(), + groups: HashMap::new(), + group_membership: HashMap::new(), }, ); let dict = FixTagLookup::new_for_tests(messages); diff --git a/src/decoder/tag_lookup.rs b/src/decoder/tag_lookup.rs index c44863e..e799454 100644 --- a/src/decoder/tag_lookup.rs +++ b/src/decoder/tag_lookup.rs @@ -14,6 +14,19 @@ pub struct MessageDef { pub _msg_type: String, pub field_order: Vec, pub required: Vec, + pub groups: HashMap, + pub group_membership: HashMap, +} + +#[derive(Debug, Clone)] +pub struct GroupSpec { + pub name: String, + pub count_tag: u32, + pub delim: u32, + pub entry_order: Vec, + pub entry_pos: HashMap, + pub entry_tag_set: HashSet, + pub nested: HashMap, } #[derive(Debug, Default, Clone)] @@ -24,6 +37,7 @@ pub struct FixTagLookup { field_types: Arc>, messages: Arc>, repeatable_tags: Arc>, + #[allow(dead_code)] trailer_order: Arc>, fallback: Option>, fallback_role: Option, @@ -77,7 +91,7 @@ impl FixTagLookup { component_map.insert(trailer.name.clone(), trailer); let messages = build_message_defs(&dict.messages, &component_map, &name_to_tag); - let repeatable_tags = collect_repeatable_tags(&dict.messages, &component_map, &name_to_tag); + let repeatable_tags = collect_repeatable_from_specs(&messages); let mut trailer_order = Vec::new(); let mut stack = Vec::new(); append_component_fields( @@ -325,10 +339,7 @@ pub fn load_dictionary_with_override(msg: &str, override_key: Option<&str>) -> A if Arc::ptr_eq(&dict, &fallback) { return dict; } - let mut merged = (*dict).clone(); - merged.fallback = Some(fallback); - merged.fallback_role = Some(FallbackKind::DetectedOverride); - let merged = Arc::new(merged); + let merged = merge_with_fallback(&dict, fallback, FallbackKind::DetectedOverride); if let Ok(mut guard) = LOOKUPS.write() { guard.insert(combo_key, merged.clone()); } @@ -347,6 +358,17 @@ fn warn_override_miss() { OVERRIDE_MISS.store(true, Ordering::Relaxed); } +fn merge_with_fallback( + primary: &Arc, + fallback: Arc, + role: FallbackKind, +) -> Arc { + let mut merged: FixTagLookup = (**primary).clone(); + merged.fallback = Some(fallback); + merged.fallback_role = Some(role); + Arc::new(merged) +} + #[cfg(test)] pub fn reset_override_warn() { OVERRIDE_MISS.store(false, Ordering::Relaxed); @@ -400,6 +422,7 @@ fn build_message_defs( let mut map = HashMap::new(); for msg in &messages.items { let (field_order, required) = expand_message_fields(msg, components, name_to_tag, true); + let (groups, membership) = collect_group_specs(&msg.groups, components, name_to_tag); map.insert( msg.msg_type.clone(), MessageDef { @@ -407,6 +430,8 @@ fn build_message_defs( _msg_type: msg.msg_type.clone(), field_order, required, + groups, + group_membership: membership, }, ); } @@ -536,44 +561,90 @@ fn dedupe(values: &mut Vec) { values.retain(|v| seen.insert(*v)); } -fn collect_repeatable_tags( - messages: &MessageContainer, +fn collect_group_specs( + groups: &[GroupDef], components: &HashMap, name_to_tag: &HashMap, -) -> HashSet { - let mut repeatable = HashSet::new(); - let mut component_stack = HashSet::new(); - - for message in &messages.items { - for component in &message.components { - collect_component_repeatables( - &component.name, - components, - name_to_tag, - &mut repeatable, - &mut component_stack, - ); +) -> (HashMap, HashMap) { + let mut specs = HashMap::new(); + let mut membership = HashMap::new(); + let mut stack = HashSet::new(); + for group in groups { + if let Some(spec) = build_group_spec(group, components, name_to_tag, &mut stack) { + membership.extend(collect_memberships(&spec, spec.count_tag)); + specs.insert(spec.count_tag, spec); } - for group in &message.groups { - collect_group_repeatables( - group, - components, - name_to_tag, - &mut repeatable, - &mut component_stack, - ); + } + // also scan groups reachable via components referenced in the message + for comp in components.values() { + for group in &comp.groups { + if let Some(spec) = build_group_spec(group, components, name_to_tag, &mut stack) { + membership.extend(collect_memberships(&spec, spec.count_tag)); + specs.entry(spec.count_tag).or_insert(spec); + } } } + (specs, membership) +} - repeatable +fn build_group_spec( + group: &GroupDef, + components: &HashMap, + name_to_tag: &HashMap, + stack: &mut HashSet, +) -> Option { + let count_tag = *name_to_tag.get(&group.name)?; + let delim = group + .fields + .first() + .and_then(|f| name_to_tag.get(&f.name)) + .copied() + .unwrap_or(count_tag); + let mut order = Vec::new(); + let mut required = Vec::new(); + append_field_refs(&group.fields, name_to_tag, &mut order, &mut required); + + let mut nested = HashMap::new(); + for comp in &group.components { + append_component_fields_for_spec( + &comp.name, + components, + name_to_tag, + stack, + &mut order, + &mut required, + &mut nested, + ); + } + for sub in &group.groups { + if let Some(spec) = build_group_spec(sub, components, name_to_tag, stack) { + order.push(spec.count_tag); + nested.insert(spec.count_tag, spec); + } + } + + dedupe(&mut order); + let entry_tag_set: HashSet = order.iter().copied().collect(); + let entry_pos: HashMap = order.iter().enumerate().map(|(i, t)| (*t, i)).collect(); + Some(GroupSpec { + name: group.name.clone(), + count_tag, + delim, + entry_order: order, + entry_pos, + entry_tag_set, + nested, + }) } -fn collect_component_repeatables( +fn append_component_fields_for_spec( name: &str, components: &HashMap, name_to_tag: &HashMap, - repeatable: &mut HashSet, stack: &mut HashSet, + order: &mut Vec, + required: &mut Vec, + nested: &mut HashMap, ) { if !stack.insert(name.to_string()) { return; @@ -583,47 +654,166 @@ fn collect_component_repeatables( return; }; - for group in &comp.groups { - collect_group_repeatables(group, components, name_to_tag, repeatable, stack); + append_field_refs(&comp.fields, name_to_tag, order, required); + for sub_comp in &comp.components { + append_component_fields_for_spec( + &sub_comp.name, + components, + name_to_tag, + stack, + order, + required, + nested, + ); } - for child in &comp.components { - collect_component_repeatables(&child.name, components, name_to_tag, repeatable, stack); + for group in &comp.groups { + if let Some(spec) = build_group_spec(group, components, name_to_tag, stack) { + order.push(spec.count_tag); + nested.insert(spec.count_tag, spec); + } } stack.remove(name); } -fn collect_group_repeatables( - group: &GroupDef, - components: &HashMap, - name_to_tag: &HashMap, - repeatable: &mut HashSet, - stack: &mut HashSet, -) { - if let Some(tag) = name_to_tag.get(&group.name) { - repeatable.insert(*tag); +fn collect_memberships(spec: &GroupSpec, owner: u32) -> HashMap { + let mut map = HashMap::new(); + for tag in &spec.entry_tag_set { + map.insert(*tag, owner); } - for field in &group.fields { - if let Some(tag) = name_to_tag.get(&field.name) { - repeatable.insert(*tag); - } + for nested in spec.nested.values() { + map.insert(nested.count_tag, nested.count_tag); + map.extend(collect_memberships(nested, nested.count_tag)); } - for comp in &group.components { - collect_component_repeatables(&comp.name, components, name_to_tag, repeatable, stack); + map +} + +fn collect_repeatable_from_specs(messages: &HashMap) -> HashSet { + fn walk(spec: &GroupSpec, acc: &mut HashSet) { + acc.insert(spec.count_tag); + for tag in &spec.entry_tag_set { + acc.insert(*tag); + } + for nested in spec.nested.values() { + walk(nested, acc); + } } - for sub in &group.groups { - collect_group_repeatables(sub, components, name_to_tag, repeatable, stack); + + let mut repeatable = HashSet::new(); + for msg in messages.values() { + for spec in msg.groups.values() { + walk(spec, &mut repeatable); + } } + repeatable } #[cfg(test)] mod tests { use super::*; + use crate::decoder::schema::FixDictionary; use once_cell::sync::Lazy; - use std::sync::Mutex; + use std::sync::{Arc, Mutex}; static LOOKUP_TEST_GUARD: Lazy> = Lazy::new(|| Mutex::new(())); + struct LookupCacheGuard { + originals: Vec<(String, Option>)>, + } + + impl LookupCacheGuard { + fn new(keys: &[&str]) -> Self { + let mut originals = Vec::new(); + if let Ok(guard) = LOOKUPS.read() { + for key in keys { + originals.push(((*key).to_string(), guard.get(*key).cloned())); + } + } else { + for key in keys { + originals.push(((*key).to_string(), None)); + } + } + Self { originals } + } + } + + impl Drop for LookupCacheGuard { + fn drop(&mut self) { + if let Ok(mut guard) = LOOKUPS.write() { + for (key, original) in &self.originals { + match original { + Some(existing) => { + guard.insert(key.clone(), existing.clone()); + } + None => { + guard.remove(key); + } + } + } + } + for (key, _) in &self.originals { + clear_override_cache_for(key); + } + } + } + + fn small_override_dictionary() -> FixDictionary { + let xml = r#" + +
+ +
+ + + + + + + + + + + + + + + + + +"#; + FixDictionary::from_xml(xml).expect("override test dictionary parses") + } + + fn small_detected_dictionary() -> FixDictionary { + let xml = r#" + +
+ +
+ + + + + + + + + + + + + + + + + + + + +"#; + FixDictionary::from_xml(xml).expect("detected test dictionary parses") + } + #[test] fn detects_schema_from_default_appl_ver_id() { let _lock = LOOKUP_TEST_GUARD.lock().unwrap(); @@ -661,7 +851,12 @@ mod tests { #[test] fn override_uses_fallback_dictionary_for_missing_tags() { let _lock = LOOKUP_TEST_GUARD.lock().unwrap(); + let _cache_guard = LookupCacheGuard::new(&["FIX44", "FIX50SP2"]); reset_override_warn(); + register_dictionary("FIX44", &small_override_dictionary()); + register_dictionary("FIX50SP2", &small_detected_dictionary()); + clear_override_cache_for("FIX44"); + clear_override_cache_for("FIX50SP2"); let msg = "8=FIXT.1.1\u{0001}35=0\u{0001}1128=9\u{0001}10=000\u{0001}"; let dict = load_dictionary_with_override(msg, Some("FIX44")); assert_eq!( @@ -674,4 +869,43 @@ mod tests { "successful fallback should not trigger override warning flag" ); } + + #[test] + fn repeatable_tags_include_nested_groups() { + let _lock = LOOKUP_TEST_GUARD.lock().unwrap(); + let xml = r#" + +
+ + + + + + + + + + + + + + + + + + + + + + + + +"#; + let dict = FixDictionary::from_xml(xml).expect("dictionary parses"); + let lookup = FixTagLookup::from_dictionary(&dict, "TEST"); + assert!(lookup.is_repeatable(900), "outer group count tag tracked"); + assert!(lookup.is_repeatable(901), "outer field repeatable"); + assert!(lookup.is_repeatable(910), "nested group count tag tracked"); + assert!(lookup.is_repeatable(911), "nested field repeatable"); + } } diff --git a/src/decoder/validator.rs b/src/decoder/validator.rs index d9ffeb6..30585ae 100644 --- a/src/decoder/validator.rs +++ b/src/decoder/validator.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: 2025 Steve Clarke - https://xyzzy.tools use crate::decoder::fixparser::{FieldValue, parse_fix}; -use crate::decoder::tag_lookup::{FixTagLookup, MessageDef}; +use crate::decoder::tag_lookup::{FixTagLookup, GroupSpec as MessageDefGroupSpec, MessageDef}; use chrono::{NaiveDate, NaiveDateTime, NaiveTime}; use once_cell::sync::Lazy; use regex::Regex; @@ -55,6 +55,12 @@ pub fn validate_fix_message(msg: &str, dict: &FixTagLookup) -> ValidationReport &msg_def.field_order, &mut tag_errors, )); + errors.extend(validate_repeating_groups( + &fields, + msg_def, + dict, + &mut tag_errors, + )); } errors.extend(validate_checksum_field(msg, &field_map, &mut tag_errors)); @@ -202,6 +208,142 @@ fn validate_field_ordering( errors } +fn validate_repeating_groups( + fields: &[FieldValue], + msg_def: &MessageDef, + dict: &FixTagLookup, + tag_errors: &mut HashMap>, +) -> Vec { + let mut errors = Vec::new(); + let mut idx = 0; + while idx < fields.len() { + let tag = fields[idx].tag; + if let Some(spec) = msg_def.groups.get(&tag) { + let (consumed, mut errs) = + validate_group_instance(fields, idx, spec, msg_def, dict, tag_errors); + errors.append(&mut errs); + idx += consumed; + } else { + if let Some(owner) = msg_def.group_membership.get(&tag) { + let err = format!( + "Tag {} ({}) appears outside of repeating group {}", + tag, + dict.field_name(tag), + owner + ); + errors.push(err.clone()); + tag_errors.entry(tag).or_default().push(err); + } + idx += 1; + } + } + errors +} + +fn validate_group_instance( + fields: &[FieldValue], + start_idx: usize, + spec: &MessageDefGroupSpec, + msg_def: &MessageDef, + dict: &FixTagLookup, + tag_errors: &mut HashMap>, +) -> (usize, Vec) { + let mut errors = Vec::new(); + let count = fields[start_idx] + .value + .parse::() + .unwrap_or_else(|_| { + let err = format!( + "Invalid NumInGroup value '{}' for tag {}", + fields[start_idx].value, spec.count_tag + ); + errors.push(err.clone()); + tag_errors + .entry(spec.count_tag) + .or_default() + .push(err.clone()); + 0 + }); + let mut entries = 0usize; + let mut idx = start_idx + 1; + + while idx < fields.len() && entries < count { + if fields[idx].tag != spec.delim { + if msg_def.group_membership.get(&fields[idx].tag) == Some(&spec.count_tag) { + let err = format!( + "Expected group delimiter tag {} before tag {}", + spec.delim, fields[idx].tag + ); + errors.push(err.clone()); + tag_errors.entry(fields[idx].tag).or_default().push(err); + idx += 1; + continue; + } else { + break; + } + } + let (consumed, mut errs) = + validate_group_entry(fields, idx, spec, msg_def, dict, tag_errors); + errors.append(&mut errs); + idx += consumed; + entries += 1; + } + + if entries != count { + let err = format!( + "NumInGroup {} declared {}, but {} instance(s) found", + spec.count_tag, count, entries + ); + errors.push(err.clone()); + tag_errors.entry(spec.count_tag).or_default().push(err); + } + (idx - start_idx, errors) +} + +fn validate_group_entry( + fields: &[FieldValue], + start_idx: usize, + spec: &MessageDefGroupSpec, + msg_def: &MessageDef, + dict: &FixTagLookup, + tag_errors: &mut HashMap>, +) -> (usize, Vec) { + let mut errors = Vec::new(); + let mut idx = start_idx; + let mut last_pos = -1isize; + while idx < fields.len() { + let tag = fields[idx].tag; + if tag == spec.delim && idx != start_idx { + break; + } + if let Some(nested) = spec.nested.get(&tag) { + let (consumed, mut errs) = + validate_group_instance(fields, idx, nested, msg_def, dict, tag_errors); + errors.append(&mut errs); + idx += consumed; + continue; + } + if let Some(pos) = spec.entry_order.iter().position(|t| *t == tag) { + if (pos as isize) < last_pos { + let err = format!( + "Tag {} ({}) out of order within repeating group {}", + tag, + dict.field_name(tag), + spec.count_tag + ); + errors.push(err.clone()); + tag_errors.entry(tag).or_default().push(err); + } + last_pos = pos as isize; + idx += 1; + } else { + // Tag does not belong to this group; stop so parent can handle it. + break; + } + } + (idx - start_idx, errors) +} + fn validate_checksum_field( msg: &str, field_map: &HashMap, diff --git a/src/main.rs b/src/main.rs index 22ade92..1b25155 100644 --- a/src/main.rs +++ b/src/main.rs @@ -756,15 +756,16 @@ fn dictionary_source(custom_dicts: &HashMap, key: &str /// Print the table header for dictionary listings. fn print_dictionary_header() { println!( - " {:<10} {:>12} {:>8} {:>11} {:>11} Source", - "Version", "ServicePack", "Fields", "Components", "Messages", + " {:<1}{:<10} {:>12} {:>8} {:>11} {:>11} Source", + "", "Version", "ServicePack", "Fields", "Components", "Messages", ); } /// Print one row of dictionary metadata. -fn print_dictionary_row(key: &str, schema: &SchemaTree, source: &str) { +fn print_dictionary_row(marker: &str, key: &str, schema: &SchemaTree, source: &str) { println!( - " {:<10} {:>12} {:>8} {:>11} {:>11} {}", + " {:<1}{:<10} {:>12} {:>8} {:>11} {:>11} {}", + marker, key, schema.service_pack, schema.fields.len(), @@ -774,6 +775,15 @@ fn print_dictionary_row(key: &str, schema: &SchemaTree, source: &str) { ); } +/// Prefix a row when the FIX key should be highlighted. +fn dictionary_marker(highlight: Option<&str>, key: &str) -> &'static str { + if matches!(highlight, Some(target) if target.eq_ignore_ascii_case(key)) { + "*" + } else { + " " + } +} + /// Determine whether a particular FIX dictionary needs the FIXT11 session /// header/trailer merged in. Saves the rest of the code from hard-coding /// these version checks repeatedly. @@ -826,8 +836,12 @@ fn key_to_xml_id(key: &str) -> Option<&'static str> { } } -/// Print a summary table of all available dictionaries (built-in and custom). -fn print_all_dictionary_info(custom_dicts: &HashMap) -> Result<()> { +/// Print a summary table of all available dictionaries (built-in and custom), +/// optionally highlighting a selected entry. +fn print_all_dictionary_info( + custom_dicts: &HashMap, + highlight: Option<&str>, +) -> Result<()> { println!( "Available FIX Dictionaries: {}", available_fix_versions(custom_dicts) @@ -839,7 +853,8 @@ fn print_all_dictionary_info(custom_dicts: &HashMap) - match load_schema_for_key(&key, custom_dicts) { Ok(schema) => { let source = dictionary_source(custom_dicts, &key); - print_dictionary_row(&key, &schema, &source); + let marker = dictionary_marker(highlight, &key); + print_dictionary_row(marker, &key, &schema, &source); } Err(err) => eprintln!("warning: failed to load {key}: {err}"), } @@ -848,26 +863,14 @@ fn print_all_dictionary_info(custom_dicts: &HashMap) - Ok(()) } -/// Handle the `--info` command, printing either all dictionaries or the selected one. +/// Handle the `--info` command, printing all dictionaries and highlighting the selected one. fn handle_info( opts: &CliOptions, - schema: &SchemaTree, + _schema: &SchemaTree, custom_dicts: &HashMap, ) -> Result<()> { - if opts.fix_from_user { - println!( - "Available FIX Dictionaries: {}", - available_fix_versions(custom_dicts) - ); - println!("\nCurrent Schema:"); - print_dictionary_header(); - let key = normalise_fix_key(&opts.fix_version).unwrap_or_else(|| "FIX44".to_string()); - let source = dictionary_source(custom_dicts, &key); - print_dictionary_row(&key, schema, &source); - println!(); - } else { - print_all_dictionary_info(custom_dicts)?; - } + let selected_key = normalise_fix_key(&opts.fix_version).unwrap_or_else(|| "FIX44".to_string()); + print_all_dictionary_info(custom_dicts, Some(&selected_key))?; Ok(()) } @@ -1161,4 +1164,11 @@ mod tests { assert!(all.contains(&"FIX44".into())); assert!(all.contains(&"FIX27".into())); } + + #[test] + fn dictionary_marker_highlights_selected_entry() { + assert_eq!(dictionary_marker(Some("fix44"), "FIX44"), "*"); + assert_eq!(dictionary_marker(Some("fix44"), "FIX50"), " "); + assert_eq!(dictionary_marker(None, "FIX44"), " "); + } }