[superseded] Fix latest main test failures

.github/workflows/ci.yml

@@ -29,10 +29,25 @@ jobs:
     # a couple of modules import pyplot at load time and would otherwise complain.
     env:
       MPLBACKEND: Agg
+      # The engine is DETERMINISTIC BY DESIGN (PYTHONHASHSEED=0, hashlib not hash(), seeded RNG). Multi-threaded BLAS
+      # breaks that: it sums float dot-products in a different ORDER per run, which flips knife-edge results (a cosine
+      # that rounds to 1.0000000000000002 instead of 1.0; a borderline z-score). So pin numpy to ONE thread and fix
+      # the hash seed -- this restores the deterministic reference the tests assume. With `-n auto` it's also FASTER:
+      # each xdist worker gets its own core with a single BLAS thread, instead of 4 workers x N BLAS threads fighting
+      # over the cores (oversubscription). If a test only fails in CI and passes locally, this is usually why.
+      PYTHONHASHSEED: "0"
+      OMP_NUM_THREADS: "1"
+      OPENBLAS_NUM_THREADS: "1"
+      MKL_NUM_THREADS: "1"
+      NUMEXPR_NUM_THREADS: "1"
 
     steps:
       - name: Check out the code
         uses: actions/checkout@v4
+        with:
+          # Full history so a pull_request can diff against its base branch's merge-base (origin/base...HEAD) to find
+          # exactly the files this PR changed. Cheap for this repo; needed by the affected-tests selection below.
+          fetch-depth: 0
 
       - name: Set up Python
         uses: actions/setup-python@v5
@@ -46,4 +61,76 @@ jobs:
           pip install -r requirements.txt
 
       - name: Run the test suite
-        run: pytest -q --durations=10
+        # Two speed levers together:
+        #   1. -n auto spreads tests across ALL the runner's CPU cores (ubuntu-latest has 4). The suite is
+        #      deterministic and each test is independent (own tmp_path / in-memory stores), so it parallelises cleanly.
+        #   2. On a PULL REQUEST we run only the tests AFFECTED by the changed files -- tools/select_tests.py builds a
+        #      static import graph and picks every test whose imports (transitively) reach a changed module. A test that
+        #      can't reach any changed file doesn't run. On a PUSH TO MAIN we run the WHOLE suite, so main is always
+        #      fully verified -- the affected-only run is fast PR feedback, the full run is the safety net.
+        # It fails safe: if a change can't be scoped (a data file, a brand-new module) select_tests prints "ALL" and we
+        # run everything; if nothing is affected (docs only) we skip the run (the doc gates below still check docs).
+        run: |
+          set -e
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            CHANGED=$(git diff --name-only "origin/${{ github.base_ref }}...HEAD")
+            echo "Changed files in this PR:"; echo "$CHANGED"
+            SELECTED=$(python tools/select_tests.py $CHANGED)
+            if [ "$SELECTED" = "ALL" ]; then
+              echo "== change can't be scoped -> running the FULL suite =="
+              pytest -q -n auto --durations=15
+            elif [ -z "$SELECTED" ]; then
+              echo "== no tests affected by these changes (docs/config only) -- skipping the test run =="
+            else
+              echo "== running only the affected tests =="; echo "$SELECTED"
+              pytest -q -n auto --durations=15 $SELECTED
+            fi
+          else
+            echo "== push to ${{ github.ref_name }} -> running the FULL suite =="
+            pytest -q -n auto --durations=15
+          fi
+
+      # Keep API_QUICKREF.md current on its own: regenerate it and fail if the committed copy is stale.
+      # This is a drift check (no write-back needed) -- if it fails, run `python apiquickref.py` locally and
+      # commit the result. The generator only reads source with `ast`, so this step is fast and side-effect-free.
+      - name: Check API_QUICKREF.md is up to date
+        run: |
+          python apiquickref.py
+          git diff --exit-code API_QUICKREF.md || {
+            echo "API_QUICKREF.md is out of date -- run 'python apiquickref.py' and commit the result.";
+            exit 1;
+          }
+
+      # DISCOVERABILITY GATE. Every capability family a user would ask for must have a curated catalog home, so the
+      # engine can surface it from a plain-English query (mind.find_capability / mind.suggest). This fails if any
+      # probe query in tools/catalog_gaps.py has no curated home -- i.e. something is built but effectively hidden.
+      # Fix by registering a home in holographic_catalog.py (see the existing ones for the shape).
+      - name: Gate -- no discoverability gaps
+        run: python tools/catalog_gaps.py
+
+      # INVOCATION GATE. Every public UnifiedMind faculty must carry a docstring an agent can act on (skill cards are
+      # built from the first line), and every "how to call it" example in a catalog home must actually resolve --
+      # both module imports and mind.method() references. This fails on a missing/thin docstring or a broken example,
+      # so a capability can never be discoverable-but-uncallable. Fix by writing the one-line docstring / correcting
+      # the example the linter names.
+      - name: Gate -- no invocation gaps
+        run: python tools/skill_lint.py
+
+      # Keep CAPABILITIES.md (the plain-language "what can it do + how to start" menu) current. Same drift-check
+      # pattern as API_QUICKREF.md: regenerate from the live catalog and fail if the committed copy is stale.
+      # capdoc.py writes no timestamp, so this only trips on a real change. Fix: `python capdoc.py` and commit.
+      - name: Check CAPABILITIES.md is up to date
+        run: |
+          python capdoc.py
+          git diff --exit-code CAPABILITIES.md || {
+            echo "CAPABILITIES.md is out of date -- run 'python capdoc.py' and commit the result.";
+            exit 1;
+          }
+
+      # ENDPOINT + FLAG GATE. SERVICE.md is mostly hand-written prose (kept), but its endpoint table must list exactly
+      # the routes the service registers, and its Launch section must mention every user-facing CLI flag the argparse
+      # defines. This checks both against the live code and fails if a route or flag was added/renamed/removed without
+      # updating the doc. It does NOT rewrite the prose; on failure it names the offender, and `python servicedoc.py
+      # --print` gives a fresh endpoint table to paste.
+      - name: Gate -- SERVICE.md documents every endpoint and flag
+        run: python servicedoc.py

.github/workflows/docs.yml

@@ -0,0 +1,42 @@
+# .github/workflows/docs.yml
+# Keep REFERENCE.md in sync with the code automatically. On every push to main, regenerate the code reference
+# from the module docstrings and commit it back if anything changed. (No third-party deps -- docgen.py is pure
+# standard library.)
+name: docs
+
+on:
+  push:
+    branches: [ main, master ]
+  workflow_dispatch:            # lets you run it by hand from the Actions tab
+
+permissions:
+  contents: write               # needed so the job can commit the refreshed REFERENCE.md back to the repo
+
+jobs:
+  reference:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Generate the code reference
+        run: python docgen.py          # writes REFERENCE.md from every module's docstring + public API
+
+      - name: Generate the capabilities menu
+        run: python capdoc.py          # writes CAPABILITIES.md from the live capability catalog (plain-language menu)
+
+      - name: Commit refreshed docs if they changed
+        run: |
+          git config user.name  "docs-bot"
+          git config user.email "docs-bot@users.noreply.github.com"
+          if ! git diff --quiet REFERENCE.md CAPABILITIES.md; then
+            git add REFERENCE.md CAPABILITIES.md
+            git commit -m "docs: refresh REFERENCE.md + CAPABILITIES.md [skip ci]"   # [skip ci] so it doesn't re-trigger CI
+            git push
+          else
+            echo "REFERENCE.md and CAPABILITIES.md already up to date -- nothing to commit."
+          fi

.github/workflows/package.yml

@@ -0,0 +1,96 @@
+# .github/workflows/package.yml
+# Build an installable leOS-core wheel on every push to main and on version tags, smoke-test that it imports,
+# upload it as a build artifact, attach it to a GitHub Release on a tag, and -- on a version tag -- publish it
+# to PyPI so people can `pip install leos-core` (and then `import lecore`).
+name: package
+
+on:
+  push:
+    branches: [ main, master ]
+    tags:     [ 'v*' ]
+  workflow_dispatch:            # lets you run it by hand from the Actions tab
+
+jobs:
+  # ---------------------------------------------------------------------------------------------------------
+  # Job 1: build the wheel + sdist, prove the wheel imports, and stash the built files as an artifact.
+  # ---------------------------------------------------------------------------------------------------------
+  build-wheel:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install the build tool
+        run: python -m pip install --upgrade pip build
+
+      # On a version tag, make sure the tag and setup.py agree BEFORE building/publishing. Catches the easy mistake
+      # of tagging v0.2.0 while setup.py still says 0.1.0 -- which would otherwise publish the wrong version (and PyPI
+      # never lets you re-upload a version). No-op on branch pushes (there's no version tag to check). Fix on failure:
+      # bump setup.py's version= to match the tag, or tag the version setup.py already has.
+      - name: Check the tag matches setup.py version
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: python tools/check_version.py --expect "${{ github.ref_name }}"
+
+      - name: Build the package (copies essentials, strips tests, builds the wheel)
+        run: sh build_package.sh
+
+      - name: Smoke-test the wheel (install it clean, in isolation, and exercise its bundled data)
+        run: |
+          python -m pip install dist/*.whl
+          # Run from a TEMP dir, not the repo checkout, so nothing resolves against repo files by accident --
+          # this proves the WHEEL is self-contained, including the vendored data it needs at runtime.
+          cd "$(mktemp -d)"
+          # The distribution is "leos-core" but the import name is still "lecore" (via the lecore.py shim).
+          python -c "import lecore; assert hasattr(lecore, 'UnifiedMind'); print('leOS-core installed; import lecore OK')"
+          # The runtime data package must be present AND its dictionary must load -- this is the check that would
+          # have caught a wheel shipped without its data. If it fails, the data isn't in package_data / build_package.sh.
+          python -c "import lecore_data; assert lecore_data.exists('knowledge', 'dictionary.json.gz'), 'vendored dictionary missing from the wheel'"
+          python -c "import holographic_dictionary as d; assert d.size() > 100000; print('dictionary works from the wheel:', d.size(), 'words')"
+
+      - name: Upload the wheel + sdist as build artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: leos-core-dist          # the later publish job downloads this exact name
+          path: dist/*
+
+      - name: Attach to the GitHub Release (only when a version tag is pushed)
+        if: startsWith(github.ref, 'refs/tags/v')
+        uses: softprops/action-gh-release@v2
+        with:
+          files: dist/*
+
+  # ---------------------------------------------------------------------------------------------------------
+  # Job 2: publish to PyPI. Runs ONLY on a version tag (v*), and ONLY after the build job succeeded.
+  #
+  # This uses PyPI "Trusted Publishing" (OIDC): GitHub proves to PyPI who it is, so there is NO API token or
+  # password stored anywhere in this repo. The `id-token: write` permission below is what lets that handshake
+  # happen. You do a ONE-TIME setup on PyPI first -- see PACKAGING.md for the exact clicks:
+  #   PyPI -> your project (or "pending publisher") -> Publishing -> add a GitHub trusted publisher with
+  #   owner = AnOversizedMooseWithSocks, repo = leCore, workflow = package.yml.
+  # After that, pushing a tag like v0.1.0 publishes that version automatically.
+  # ---------------------------------------------------------------------------------------------------------
+  publish-pypi:
+    needs: build-wheel                       # don't publish unless the wheel built and imported cleanly
+    if: startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write                        # REQUIRED for trusted publishing (the OIDC handshake). No token needed.
+    # environment: pypi                      # optional: create a GitHub Environment named "pypi" for an extra
+                                             #   approval gate, then set the same name on PyPI's publisher config
+                                             #   and uncomment this line. Leave commented for the simplest setup.
+    steps:
+      - name: Download the built wheel + sdist from Job 1
+        uses: actions/download-artifact@v4
+        with:
+          name: leos-core-dist
+          path: dist                         # the publish action expects the files in dist/
+
+      - name: Publish to PyPI (trusted publishing -- no stored secret)
+        uses: pypa/gh-action-pypi-publish@release/v1
+        # Defaults publish to the real PyPI. To rehearse on TestPyPI first, uncomment:
+        # with:
+        #   repository-url: https://test.pypi.org/legacy/

.gitignore

@@ -1,2 +1,18 @@
 /.venv
 /__pycache__
+holostuff.zip
+/current backlogs
+holostuff_chunking_transfer_backlog.md
+holostuff_consolidation_backlog.md
+holostuff_crosscutting_backlog.md
+holostuff_holographic_dcc_backlog.md
+holostuff_inverse_rendering_backlog.md
+holostuff_panel_generative_compression.md
+holostuff_panel_integration_review.md
+holostuff_panel_proven_structure.md
+holostuff_panel_review_crosscutting.md
+holostuff_panel_review_rendering_backlog.md
+holostuff_rendering_backlog.md
+holostuff_vm_backlog.md
+PANEL_occlusion_speed_backlog.md
+RENDER_PIPELINE_BACKLOG.md