diff --git a/.bandit-baseline.json b/.bandit-baseline.json new file mode 100644 index 0000000..4005d4e --- /dev/null +++ b/.bandit-baseline.json @@ -0,0 +1,683 @@ +{ + "errors": [], + "generated_at": "2026-07-20T23:16:14Z", + "metrics": { + "_totals": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 10892, + "nosec": 0, + "skipped_tests": 18 + }, + "cli/__init__.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 0, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\__init__.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 0, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\analyzers\\__init__.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 28, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\analyzers\\base.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 18, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\analyzers\\csharp_analyzer.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 689, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\analyzers\\csharp_ast_bridge.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 138, + "nosec": 0, + "skipped_tests": 3 + }, + "cli/ucli\\analyzers\\go_analyzer.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 454, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\analyzers\\go_ast_bridge.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 114, + "nosec": 0, + "skipped_tests": 2 + }, + "cli/ucli\\analyzers\\java_analyzer.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 603, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\analyzers\\js_analyzer.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 670, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\analyzers\\js_ast_bridge.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 123, + "nosec": 0, + "skipped_tests": 2 + }, + "cli/ucli\\analyzers\\map_meta.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 97, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\analyzers\\python_analyzer.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 1498, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\analyzers\\registry.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 246, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\analyzers\\rust_analyzer.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 416, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\analyzers\\rust_ast_bridge.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 113, + "nosec": 0, + "skipped_tests": 2 + }, + "cli/ucli\\boundaries\\scan.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 31, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\commands\\__init__.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 5, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\commands\\ci_ops.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 182, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\commands\\contracts_ops.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 90, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\commands\\demo_ops.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 59, + "nosec": 0, + "skipped_tests": 2 + }, + "cli/ucli\\commands\\diff_ops.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 480, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\commands\\doctor_ops.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 205, + "nosec": 0, + "skipped_tests": 3 + }, + "cli/ucli\\commands\\explorer_ops.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 444, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\commands\\init_ops.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 697, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\commands\\lens_ops.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 208, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\commands\\metrics_ops.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 175, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\commands\\pack_ops.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 15, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\commands\\report_ops.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 81, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\commands\\scan_ops.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 287, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\commands\\tour_ops.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 93, + "nosec": 0, + "skipped_tests": 2 + }, + "cli/ucli\\commands\\trace_ops.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 30, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\commands\\typer_groups.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 380, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\commands\\visual_ops.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 20, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\commands\\wizard_ops.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 292, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\config.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 84, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\contracts\\contracts.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 425, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\contracts\\lang_strategies.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 71, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\dashboard\\build.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 34, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\glossary\\build.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 37, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\graph\\graph.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 12, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\lens\\ingest.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 11, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\lens\\lens.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 193, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\main.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 47, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\metrics\\analytics.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 514, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\metrics\\ttu.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 44, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\pack\\publish.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 34, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\packs\\pack.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 6, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\report\\report.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 49, + "nosec": 0, + "skipped_tests": 0 + }, + "cli/ucli\\trace\\pytrace.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 321, + "nosec": 0, + "skipped_tests": 2 + }, + "cli/ucli\\visual\\delta.py": { + "CONFIDENCE.HIGH": 0, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 0, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 29, + "nosec": 0, + "skipped_tests": 0 + } + }, + "results": [] +} \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a59a71..a9f68e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -160,32 +160,8 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max - publish: - needs: [test, lint, build] - runs-on: ubuntu-latest - if: github.event_name == 'release' - - steps: - - uses: actions/checkout@v4 - - - name: Download build artifacts - uses: actions/download-artifact@v4 - with: - name: dist - path: dist/ - - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install twine - run: pip install twine - - - name: Publish to PyPI - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} - run: twine upload dist/* + # PyPI publish lives solely in publish-pypi.yml (trusted publishing). + # Dual upload from this workflow raced on release and is intentionally omitted. security: runs-on: ubuntu-latest @@ -200,11 +176,23 @@ jobs: - name: Install bandit run: pip install bandit - - name: Run bandit security linter - run: bandit -r cli/ -f json -o bandit-report.json || true + # Fail-closed security policy: + # 1) Any HIGH severity finding fails the job (none expected today). + # 2) Any finding not in the committed .bandit-baseline.json also fails + # (catches new LOW/MEDIUM drift). Update the baseline deliberately when + # accepting a finding — do not silence with continue-on-error. + - name: Run bandit (fail on HIGH) + run: bandit -r cli/ -lll -f screen + + - name: Run bandit (fail on new vs baseline) + run: bandit -r cli/ -b .bandit-baseline.json -f json -o bandit-report.json - name: Upload security report + if: always() uses: actions/upload-artifact@v4 with: name: security-report path: bandit-report.json + if-no-files-found: warn + + diff --git a/.github/workflows/pack-publish.yml b/.github/workflows/pack-publish.yml index 7be72b0..5b6fe25 100644 --- a/.github/workflows/pack-publish.yml +++ b/.github/workflows/pack-publish.yml @@ -14,18 +14,25 @@ jobs: - run: | python -m pip install --upgrade pip python -m pip install -e ".[dev,examples]" - u pack --publish + u pack publish - name: Prepare site run: | - mkdir -p site + mkdir -p site/docs echo "

Understanding Pack

" >> site/index.html - uses: actions/upload-pages-artifact@v3 - with: { path: "." } + with: { path: "site" } deploy: needs: build permissions: diff --git a/.github/workflows/pr-delta-comment.yml b/.github/workflows/pr-delta-comment.yml index 6853aa2..c7b1c75 100644 --- a/.github/workflows/pr-delta-comment.yml +++ b/.github/workflows/pr-delta-comment.yml @@ -1,39 +1,22 @@ -name: PR Delta Comment +# DISABLED (Wave 2 consolidation): Soft-failed every step (`|| true`) and duplicated +# the thin PR map-diff path. Prefer: +# .github/workflows/understand-first-pr-analysis.yml +name: PR Delta Comment (disabled) + on: - pull_request: - types: [opened, synchronize] + workflow_dispatch: + inputs: + reason: + description: "Disabled — use understand-first-pr-analysis.yml" + required: false + default: "manual-only" + jobs: - delta: + disabled: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Install CLI + - name: Workflow disabled pending consolidation run: | - python -m pip install --upgrade pip - python -m pip install -e ".[dev,examples]" - - name: Build lens (base) - run: | - git fetch origin ${{ github.base_ref }} --depth=1 || true - git checkout -qf ${{ github.base_ref }} || true - u scan . -o maps/repo_base.json || true - u lens from-seeds --seed $(git ls-files | head -n 30) --map maps/repo_base.json -o maps/lens_base.json || true - - name: Build lens (head) - run: | - git checkout -qf ${{ github.sha }} - u scan . -o maps/repo_head.json || true - u lens from-seeds --seed $(git diff --name-only ${{ github.base_ref }} | tr '\n' ' ') --map maps/repo_head.json -o maps/lens_head.json || true - - name: Visualize delta - run: | - u visual delta maps/lens_base.json maps/lens_head.json -o maps/delta.svg || true - - name: Comment on PR with delta - uses: actions/github-script@v7 - with: - script: | - const fs = require('fs'); - let svg = ''; try { svg = fs.readFileSync('maps/delta.svg', 'utf8'); } catch(e) {} - const b64 = Buffer.from(svg).toString('base64'); - const body = `### Understand-First Delta\ndelta`; - github.rest.issues.createComment({ ...context.repo, issue_number: context.issue.number, body }); + echo "pr-delta-comment.yml is intentionally disabled." + echo "Use understand-first-pr-analysis.yml instead." + exit 0 diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index b87a4a6..dbe973c 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -1,4 +1,5 @@ name: Publish PyPI +# Sole PyPI publisher (OIDC trusted publishing). Do not re-add twine upload in ci.yml. on: workflow_dispatch: {} release: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8bbf3d7..192adab 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -34,10 +34,11 @@ jobs: echo "" >> dist/RELEASE_NOTES.md echo "## VSIX SHA256" >> dist/RELEASE_NOTES.md cat dist/VSIX_SHA256.txt >> dist/RELEASE_NOTES.md - - name: Build understanding pack + - name: Build understanding pack (optional release asset) + continue-on-error: true run: | python -m pip install -e ".[dev,examples]" - u pack --publish || true + u pack publish - name: Upload artifacts uses: softprops/action-gh-release@v2 with: diff --git a/.github/workflows/tour-must-pass.yml b/.github/workflows/tour-must-pass.yml index 916eae7..c5ce7b3 100644 --- a/.github/workflows/tour-must-pass.yml +++ b/.github/workflows/tour-must-pass.yml @@ -1,3 +1,5 @@ +# Fail-closed tour gate (Wave 3): critical scan/lens/trace/tour steps must succeed. +# Soft-fail only optional gRPC codegen (HTTP server is required for hot_path). name: Tour Must Pass on: pull_request: @@ -7,39 +9,78 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + fetch-depth: 2 + - uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: "3.11" + - name: Install Python deps run: | python -m pip install --upgrade pip python -m pip install -e ".[dev,examples]" + - name: Build repo map run: | - u scan . -o maps/repo.json || true + mkdir -p maps tours traces + u scan . -o maps/repo.json + - name: Create lens from PR files run: | - CHANGED=$(git diff --name-only HEAD~1 | tr '\n' ' ') - u lens from-seeds --seed $CHANGED --map maps/repo.json -o maps/lens.json || true + CHANGED=$(git diff --name-only HEAD~1 2>/dev/null | tr '\n' ' ' || true) + if [ -z "$(echo "$CHANGED" | tr -d '[:space:]')" ]; then + CHANGED="examples/app/hot_path.py" + fi + u lens from-seeds --seed $CHANGED --map maps/repo.json -o maps/lens.json + test -f maps/lens.json + - name: Generate gRPC Python code + continue-on-error: true run: | - python -m grpc_tools.protoc -I examples/apis --python_out=examples/apis/gen --grpc_python_out=examples/apis/gen examples/apis/orders.proto + python -m grpc_tools.protoc -I examples/apis \ + --python_out=examples/apis/gen \ + --grpc_python_out=examples/apis/gen \ + examples/apis/orders.proto + - name: Start HTTP server run: | nohup python examples/servers/http_server.py >/tmp/http.log 2>&1 & sleep 2 + curl -sf http://127.0.0.1:8000/pets >/dev/null + - name: Start gRPC mock + continue-on-error: true run: | nohup python examples/servers/grpc_server.py >/tmp/grpc.log 2>&1 & sleep 2 + - name: Trace hot path (runtime truth) run: | u trace module examples/app/hot_path.py run_hot_path -o traces/tour.json + test -f traces/tour.json u lens merge-trace maps/lens.json traces/tour.json -o maps/lens_merged.json + test -f maps/lens_merged.json + - name: Generate tour (from merged lens) run: | - u tour maps/lens_merged.json -o tours/PR.md || true - - name: Run tour (fixture) + u tour maps/lens_merged.json -o tours/PR.md + test -f tours/PR.md + + - name: Run tour fixture (fail-closed gate) run: | - set -e u tour_run maps/lens_merged.json -f fixtures + + - name: Upload tour artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: tour-must-pass-artifacts + path: | + maps/repo.json + maps/lens.json + maps/lens_merged.json + traces/tour.json + tours/PR.md + retention-days: 14 + if-no-files-found: warn diff --git a/.github/workflows/understand-first-commenter.yml b/.github/workflows/understand-first-commenter.yml index 826905a..1b84a3c 100644 --- a/.github/workflows/understand-first-commenter.yml +++ b/.github/workflows/understand-first-commenter.yml @@ -1,201 +1,25 @@ -name: Understand-First Commenter Service +# DISABLED (P0-01 / P1-06): Inline commenter invoked non-existent `u analyze` +# and installed only loose pip deps instead of the understand-first package. +# +# Active thin PR path: +# .github/workflows/understand-first-pr-analysis.yml +# (pip install -e . ; u scan ; u diff --old/--new) +name: Understand-First Commenter Service (disabled) on: - pull_request: - types: [opened, synchronize, reopened, closed] workflow_dispatch: inputs: - pr_number: - description: "PR number to analyze" - required: true - type: string + reason: + description: "Disabled workflow — see file header TODO" + required: false + default: "manual-only" jobs: - commenter-service: + disabled: runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: write - checks: write - actions: read - statuses: write - issues: write - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install typer rich aiohttp asyncpg pydantic requests - - - name: Run Understand-First Commenter - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_NUMBER: ${{ github.event.number || github.event.inputs.pr_number }} + - name: Workflow disabled pending real CLI rewrite run: | - python -c " - import os - import json - import requests - from datetime import datetime - - # GitHub API setup - token = os.environ['GITHUB_TOKEN'] - pr_number = os.environ['PR_NUMBER'] - repo = os.environ['GITHUB_REPOSITORY'] - - headers = { - 'Authorization': f'token {token}', - 'Accept': 'application/vnd.github.v3+json' - } - - # Get PR details - pr_url = f'https://api.github.com/repos/{repo}/pulls/{pr_number}' - pr_response = requests.get(pr_url, headers=headers) - pr_data = pr_response.json() - - base_sha = pr_data['base']['sha'] - head_sha = pr_data['head']['sha'] - - print(f'Analyzing PR #{pr_number}: {pr_data[\"title\"]}') - print(f'Base: {base_sha[:8]} -> Head: {head_sha[:8]}') - - # Run analysis on both commits - os.system(f'u analyze --output base-analysis.json --format json --commit {base_sha}') - os.system(f'u analyze --output head-analysis.json --format json --commit {head_sha}') - - # Generate delta analysis - os.system('u diff base-analysis.json head-analysis.json --output pr-delta.json --format json --policy-check --verbose') - - # Generate comment content - os.system('u diff base-analysis.json head-analysis.json --output pr-comment.md --format markdown --policy-check') - - # Generate mini-map - os.system('u diff base-analysis.json head-analysis.json --output pr-minimap.svg --format svg') - - # Read results - with open('pr-delta.json', 'r') as f: - delta_data = json.load(f) - - with open('pr-comment.md', 'r') as f: - comment_content = f.read() - - # Read mini-map - with open('pr-minimap.svg', 'r') as f: - minimap_svg = f.read() - - # Create enhanced comment - violations = delta_data.get('violations', []) - policy_passed = len(violations) == 0 - - # Convert SVG to data URL - import base64 - minimap_base64 = base64.b64encode(minimap_svg.encode()).decode() - minimap_data_url = f'data:image/svg+xml;base64,{minimap_base64}' - - enhanced_comment = f''' - ## 🔍 Understand-First PR Analysis - - {comment_content} - - ### 📊 Mini-Map Visualization - \"Code - - ### ✅ Policy Compliance Checklist - {('✅ **All policies passed!** No violations detected.' if policy_passed else - '\\n'.join([f'❌ **{v[\"rule\"]}**: {v[\"description\"]}' for v in violations]))} - - ### 📈 Key Metrics - - **Functions Added**: {len(delta_data.get('functions_added', []))} - - **Functions Removed**: {len(delta_data.get('functions_removed', []))} - - **Functions Modified**: {len(delta_data.get('functions_modified', []))} - - **Complexity Change**: {delta_data.get('complexity_change', 0):+d} - - **Side Effects**: {len(delta_data.get('side_effects_added', []))} added, {len(delta_data.get('side_effects_removed', []))} removed - - ### 🎯 Action Items - {('✅ Ready to merge!' if policy_passed else - '\\n'.join([f'- [ ] Fix: {v[\"description\"]} in `{v[\"function_name\"]}`' for v in violations]))} - - --- - *Generated by [Understand-First](https://github.com/your-org/understand-first) • [View full analysis]({pr_data[\"html_url\"]}/checks)* - ''' - - # Post or update comment - comments_url = f'https://api.github.com/repos/{repo}/issues/{pr_number}/comments' - comments_response = requests.get(comments_url, headers=headers) - comments = comments_response.json() - - existing_comment = next((c for c in comments if 'Understand-First PR Analysis' in c['body']), None) - - if existing_comment: - # Update existing comment - update_url = f'https://api.github.com/repos/{repo}/issues/comments/{existing_comment[\"id\"]}' - requests.patch(update_url, headers=headers, json={'body': enhanced_comment}) - print('Updated existing comment') - else: - # Create new comment - requests.post(comments_url, headers=headers, json={'body': enhanced_comment}) - print('Created new comment') - - # Create Check Run - check_run_data = { - 'name': 'Understand-First Analysis', - 'head_sha': head_sha, - 'status': 'completed', - 'conclusion': 'success' if policy_passed else 'failure', - 'output': { - 'title': '✅ Understand-First Policy Check Passed' if policy_passed else '❌ Understand-First Policy Check Failed', - 'summary': 'All code quality policies passed. The PR is safe to merge.' if policy_passed else f'{len(violations)} policy violations detected. Please review before merging.', - 'text': '\\n\\n'.join([f'**{v[\"rule\"]}**: {v[\"description\"]}\\n- Function: {v[\"function_name\"]}\\n- Severity: {v[\"severity\"]}\\n- File: {v[\"file_path\"]}:{v[\"line_number\"]}' for v in violations]) if violations else 'No violations found.' - }, - 'actions': [ - { - 'label': 'View Full Report', - 'description': 'Open detailed analysis report', - 'identifier': 'view-report' - }, - { - 'label': 'Download Artifacts', - 'description': 'Download analysis artifacts', - 'identifier': 'download-artifacts' - } - ] - } - - checks_url = f'https://api.github.com/repos/{repo}/check-runs' - requests.post(checks_url, headers=headers, json=check_run_data) - print('Created Check Run') - - # Set commit status - status_data = { - 'state': 'success' if policy_passed else 'failure', - 'description': 'Understand-First: All policies passed' if policy_passed else 'Understand-First: Policy violations detected', - 'context': 'understand-first/policy-check' - } - - status_url = f'https://api.github.com/repos/{repo}/statuses/{head_sha}' - requests.post(status_url, headers=headers, json=status_data) - print('Set commit status') - - print(f'Analysis complete. Policy check: {\"PASSED\" if policy_passed else \"FAILED\"}') - " - - - name: Upload analysis artifacts - uses: actions/upload-artifact@v3 - with: - name: understand-first-pr-analysis-${{ github.event.number || github.event.inputs.pr_number }} - path: | - base-analysis.json - head-analysis.json - pr-delta.json - pr-comment.md - pr-minimap.svg + echo "understand-first-commenter.yml is intentionally disabled." + echo "Install the package and use u scan / u diff --old/--new if re-enabling." + exit 0 diff --git a/.github/workflows/understand-first-pr-analysis.yml b/.github/workflows/understand-first-pr-analysis.yml index 6ae4116..e867961 100644 --- a/.github/workflows/understand-first-pr-analysis.yml +++ b/.github/workflows/understand-first-pr-analysis.yml @@ -1,243 +1,141 @@ +# Thin PR analysis path (Wave 2/3): +# pip install -e . +# u scan . -o maps/repo_{base,head}.json +# u diff --old ... --new ... -o maps/delta.json --json +# +# Install notes: +# - Default `pip install -e .` is enough for Python AST maps (McCabe) and for +# JS/TS/Go/Rust/C# regex best-effort degrade paths (no extra toolchains). +# - Optional: `pip install -e ".[analyzers]"` adds javalang for Java AST when +# present; without it, Java scans degrade to keyword-heuristic regex. +# - Optional toolchains (Node/Go/Rust/dotnet) unlock JS/Go/Rust/C# AST workers +# when present on the runner; this workflow does not install them, so +# non-Python languages typically stay on regex degrade — that is intentional. +# +# Complexity delta comes from real McCabe scores on Python maps. +# PR comment is fail-soft (permissions); scan/diff are fail-closed. name: Understand-First PR Analysis on: pull_request: types: [opened, synchronize, reopened] - workflow_dispatch: + +permissions: + contents: read + pull-requests: write jobs: - analyze-pr: + pr-map-diff: runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: write - checks: write - actions: read - statuses: write - issues: write - steps: - - name: Checkout code + - name: Checkout PR head uses: actions/checkout@v4 with: - fetch-depth: 0 # Full history for diff analysis + fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.11" - - name: Install dependencies + - name: Install Understand-First run: | python -m pip install --upgrade pip - python -m pip install -e ".[dev,examples]" - - - name: Run understand-first analysis on base commit - id: analyze-base - run: | - echo "Analyzing base commit..." - u analyze --output base-analysis.json --format json - echo "base_analysis_file=base-analysis.json" >> $GITHUB_OUTPUT - - - name: Run understand-first analysis on PR head - id: analyze-head - run: | - echo "Analyzing PR head commit..." - u analyze --output head-analysis.json --format json - echo "head_analysis_file=head-analysis.json" >> $GITHUB_OUTPUT + python -m pip install -e . - - name: Generate delta analysis - id: delta-analysis + - name: Scan PR head run: | - echo "Generating delta analysis..." - u diff base-analysis.json head-analysis.json \ - --output pr-delta.json \ - --format json \ - --policy-check \ - --verbose - echo "delta_file=pr-delta.json" >> $GITHUB_OUTPUT + mkdir -p maps + u scan . -o maps/repo_head.json - - name: Generate PR comment - id: pr-comment + - name: Scan PR base run: | - echo "Generating PR comment..." - u diff base-analysis.json head-analysis.json \ - --output pr-comment.md \ - --format markdown \ - --policy-check - echo "comment_file=pr-comment.md" >> $GITHUB_OUTPUT + BASE_SHA="${{ github.event.pull_request.base.sha }}" + git checkout --force "$BASE_SHA" + u scan . -o maps/repo_base.json + git checkout --force "${{ github.sha }}" - - name: Generate mini-map visualization - id: minimap + - name: Diff base vs head maps run: | - echo "Generating mini-map..." - u diff base-analysis.json head-analysis.json \ - --output pr-minimap.svg \ - --format svg - echo "minimap_file=pr-minimap.svg" >> $GITHUB_OUTPUT + u diff --old maps/repo_base.json --new maps/repo_head.json -o maps/delta.json --json - - name: Check policy compliance - id: policy-check + - name: Summarize delta run: | - echo "Checking policy compliance..." - u diff base-analysis.json head-analysis.json \ - --policy-check \ - --output policy-report.json \ - --format json - POLICY_EXIT_CODE=$? - echo "policy_exit_code=$POLICY_EXIT_CODE" >> $GITHUB_OUTPUT - echo "Policy check exit code: $POLICY_EXIT_CODE" - - - name: Upload analysis artifacts - uses: actions/upload-artifact@v3 + echo "## Understand-First map delta" >> "$GITHUB_STEP_SUMMARY" + jq -r ' + "### Summary", + "- Added: \(.summary.added // 0)", + "- Removed: \(.summary.removed // 0)", + "- Modified: \(.summary.modified // 0)", + "- Policy breaches: \(.summary.policy_breaches // 0)", + "- Complexity net change: \(.summary.complexity_net_change // .complexity_delta.net_change // 0)", + "- Side-effect tags added (heuristic): \(.summary.side_effects_added // 0)", + "", + "Complexity = McCabe on Python AST maps; JS/TS maps use keyword-heuristic complexity (not McCabe). Side effects = Python AST heuristics when present." + ' maps/delta.json >> "$GITHUB_STEP_SUMMARY" + + - name: Upload map artifacts + uses: actions/upload-artifact@v4 with: - name: understand-first-analysis + name: understand-first-pr-maps path: | - base-analysis.json - head-analysis.json - pr-delta.json - pr-comment.md - pr-minimap.svg - policy-report.json - - - name: Post PR comment - uses: actions/github-script@v6 + maps/repo_base.json + maps/repo_head.json + maps/delta.json + retention-days: 14 + + - name: Comment delta summary on PR + if: github.event_name == 'pull_request' + continue-on-error: true + uses: actions/github-script@v7 with: script: | const fs = require('fs'); - const path = require('path'); - - // Read the generated comment - const commentContent = fs.readFileSync('${{ steps.pr-comment.outputs.comment_file }}', 'utf8'); - - // Read policy report - const policyReport = JSON.parse(fs.readFileSync('policy-report.json', 'utf8')); - - // Create enhanced comment with mini-map - const minimapBase64 = fs.readFileSync('${{ steps.minimap.outputs.minimap_file }}', 'base64'); - const minimapDataUrl = `data:image/svg+xml;base64,${minimapBase64}`; - - const enhancedComment = ` - ## 🔍 Understand-First PR Analysis - - ${commentContent} - - ### 📊 Mini-Map Visualization - Code Map Delta - - ### ✅ Policy Compliance Checklist - ${policyReport.violations.length === 0 ? - '✅ **All policies passed!** No violations detected.' : - policyReport.violations.map(v => `❌ **${v.rule}**: ${v.description}`).join('\n') + let delta; + try { + delta = JSON.parse(fs.readFileSync('maps/delta.json', 'utf8')); + } catch (e) { + core.warning('delta.json missing; skipping PR comment'); + return; } - - ### 📈 Key Metrics - - **Functions Added**: ${policyReport.delta.functions_added.length} - - **Functions Removed**: ${policyReport.delta.functions_removed.length} - - **Functions Modified**: ${policyReport.delta.functions_modified.length} - - **Complexity Change**: ${policyReport.delta.complexity_change > 0 ? '+' : ''}${policyReport.delta.complexity_change} - - **Side Effects**: ${policyReport.delta.side_effects_added.length} added, ${policyReport.delta.side_effects_removed.length} removed - - --- - *Generated by [Understand-First](https://github.com/your-org/understand-first) • [View full analysis](${context.payload.pull_request.html_url}/checks)* - `; - - // Find existing comment - const comments = await github.rest.issues.listComments({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - }); - - const existingComment = comments.data.find(comment => - comment.user.type === 'Bot' && - comment.body.includes('Understand-First PR Analysis') + const s = delta.summary || {}; + const cd = delta.complexity_delta || {}; + const net = s.complexity_net_change ?? cd.net_change ?? 0; + const increased = (cd.increased || []).slice(0, 8); + const breachLines = (delta.policy_breaches || []).slice(0, 8).map( + b => `- \`${b.function}\`: ${b.old_complexity} → ${b.new_complexity} (threshold ${b.threshold})` ); - - if (existingComment) { - // Update existing comment - await github.rest.issues.updateComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: existingComment.id, - body: enhancedComment - }); - } else { - // Create new comment - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body: enhancedComment - }); - } - - - name: Create Check Run - uses: actions/github-script@v6 - with: - script: | - const fs = require('fs'); - const policyReport = JSON.parse(fs.readFileSync('policy-report.json', 'utf8')); - const policyExitCode = ${{ steps.policy-check.outputs.policy_exit_code }}; - - const conclusion = policyExitCode === 0 ? 'success' : 'failure'; - const title = policyExitCode === 0 ? - '✅ Understand-First Policy Check Passed' : - '❌ Understand-First Policy Check Failed'; - - const summary = policyExitCode === 0 ? - 'All code quality policies passed. The PR is safe to merge.' : - `${policyReport.violations.length} policy violations detected. Please review before merging.`; - - const details = policyReport.violations.map(v => - `**${v.rule}**: ${v.description}\n` + - `- Function: ${v.function_name}\n` + - `- Severity: ${v.severity}\n` + - `- File: ${v.file_path}:${v.line_number}` - ).join('\n\n'); - - await github.rest.checks.create({ - owner: context.repo.owner, - repo: context.repo.repo, - name: 'Understand-First Analysis', - head_sha: context.sha, - status: 'completed', - conclusion: conclusion, - output: { - title: title, - summary: summary, - text: details || 'No violations found.' - }, - actions: [ - { - label: 'View Full Report', - description: 'Open detailed analysis report', - identifier: 'view-report' - }, - { - label: 'Download Artifacts', - description: 'Download analysis artifacts', - identifier: 'download-artifacts' - } - ] - }); - - - name: Set commit status - if: always() - uses: actions/github-script@v6 - with: - script: | - const policyExitCode = ${{ steps.policy-check.outputs.policy_exit_code }}; - const state = policyExitCode === 0 ? 'success' : 'failure'; - const description = policyExitCode === 0 ? - 'Understand-First: All policies passed' : - 'Understand-First: Policy violations detected'; - - await github.rest.repos.createCommitStatus({ + const body = [ + '## Understand-First PR map delta', + '', + '| Metric | Value |', + '|--------|------:|', + `| Functions added | ${s.added ?? 0} |`, + `| Functions removed | ${s.removed ?? 0} |`, + `| Functions modified | ${s.modified ?? 0} |`, + `| Complexity net change | ${net >= 0 ? '+' : ''}${net} |`, + `| Complexity total (base → head) | ${cd.total_old ?? s.complexity_total_old ?? '?'} → ${cd.total_new ?? s.complexity_total_new ?? '?'} |`, + `| Side-effect tag adds (heuristic) | ${s.side_effects_added ?? 0} |`, + `| Policy breaches | ${s.policy_breaches ?? 0} |`, + '', + increased.length + ? '### Complexity increases\n' + increased.map( + x => `- \`${x.function}\`: ${x.old_complexity} → ${x.new_complexity}` + ).join('\n') + : '### Complexity increases\n_None_', + '', + breachLines.length + ? '### Policy breaches\n' + breachLines.join('\n') + : '', + '', + '_Generated by `u scan` + `u diff --old/--new` (Python AST + JS/TS best-effort when present)._', + '_Complexity: McCabe for Python; keyword-heuristic for JS/TS — mixed totals are not uniform McCabe._', + '_Side effects are Python AST heuristics when present; JS maps typically omit them._', + '_PR comment step is fail-soft; scan/diff above are fail-closed._', + ].filter(Boolean).join('\n'); + github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, - sha: context.sha, - state: state, - description: description, - context: 'understand-first/policy-check' + issue_number: context.issue.number, + body, }); diff --git a/.github/workflows/understand-first.yml b/.github/workflows/understand-first.yml index 8932f11..7bff73b 100644 --- a/.github/workflows/understand-first.yml +++ b/.github/workflows/understand-first.yml @@ -1,415 +1,25 @@ -name: Understand-First Analysis +# DISABLED (Wave 2 consolidation): Bloated PR job overclaimed side_effects / lens / +# tour / dashboard paths and mixed soft-fail theater with critical analysis. +# +# Active thin PR analysis lives in: +# .github/workflows/understand-first-pr-analysis.yml +# (u scan + u diff --old/--new only; package installed via pip install -e .) +name: Understand-First Analysis (disabled) on: - pull_request: - types: [opened, synchronize, reopened] - push: - branches: [main, master] + workflow_dispatch: + inputs: + reason: + description: "Disabled — use understand-first-pr-analysis.yml" + required: false + default: "manual-only" jobs: - understand-analysis: + disabled: runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Fetch full history for better analysis - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - - name: Install Understand-First - run: | - python -m pip install --upgrade pip - python -m pip install -e ".[dev,examples]" - - - name: Scan Repository - id: scan - run: | - echo "🔍 Scanning repository for code understanding..." - u scan . -o maps/repo.json --verbose - echo "functions_count=$(jq '.functions | length' maps/repo.json)" >> $GITHUB_OUTPUT - echo "modules_count=$(jq -r '.functions | to_entries | map(.value.file) | unique | length' maps/repo.json)" >> $GITHUB_OUTPUT - - - name: Generate Understanding Lens - id: lens + - name: Workflow disabled in favor of thin PR analysis run: | - echo "🎯 Creating understanding lens for changed files..." - - # Get changed files from PR - if [ "${{ github.event_name }}" = "pull_request" ]; then - CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD | grep -E '\.(py|ts|js|go|rs|java)$' | head -10) - else - CHANGED_FILES=$(git diff --name-only HEAD~1...HEAD | grep -E '\.(py|ts|js|go|rs|java)$' | head -10) - fi - - echo "Changed files: $CHANGED_FILES" - - if [ -n "$CHANGED_FILES" ]; then - # Create lens from changed files - u lens from-seeds --map maps/repo.json --seed $CHANGED_FILES -o maps/lens.json --verbose - echo "lens_created=true" >> $GITHUB_OUTPUT - echo "lens_functions=$(jq '.functions | length' maps/lens.json)" >> $GITHUB_OUTPUT - else - echo "lens_created=false" >> $GITHUB_OUTPUT - echo "lens_functions=0" >> $GITHUB_OUTPUT - fi - - - name: Generate Understanding Tour - if: steps.lens.outputs.lens_created == 'true' - run: | - echo "📚 Generating understanding tour..." - u tour maps/lens.json -o tours/pr-tour.md - - - name: Generate Context Debt Dashboard - if: steps.lens.outputs.lens_created == 'true' - run: | - echo "📊 Generating context debt dashboard..." - u ci --scan . --output ci-artifacts --report - - # Generate dashboard HTML - cat > context-debt-dashboard.html << 'EOF' - - - - Context Debt Dashboard - - - -

🧠 Context Debt Dashboard

-
-
-

📊 Code Complexity

-
Loading...
-
Average function complexity
-
-
-

⚠️ Side Effects

-
Loading...
-
Functions with side effects
-
-
-

🔗 Dependencies

-
Loading...
-
Function dependencies
-
-
-

📈 Understanding Score

-
Loading...
-
Overall code understandability
-
-
- - - - EOF - - - name: Analyze Code Changes - id: diff - if: steps.lens.outputs.lens_created == 'true' - run: | - echo "📊 Analyzing code changes..." - - # Create baseline lens from previous commit - git stash - git checkout HEAD~1 - u scan . -o maps/repo_old.json --verbose || echo "No previous commit to compare" - - if [ -f "maps/repo_old.json" ]; then - u diff --old maps/repo_old.json --new maps/repo.json -o maps/delta.json --json - echo "changes_detected=true" >> $GITHUB_OUTPUT - echo "added_functions=$(jq '.summary.added' maps/delta.json)" >> $GITHUB_OUTPUT - echo "removed_functions=$(jq '.summary.removed' maps/delta.json)" >> $GITHUB_OUTPUT - echo "modified_functions=$(jq '.summary.modified' maps/delta.json)" >> $GITHUB_OUTPUT - else - echo "changes_detected=false" >> $GITHUB_OUTPUT - echo "added_functions=0" >> $GITHUB_OUTPUT - echo "removed_functions=0" >> $GITHUB_OUTPUT - echo "modified_functions=0" >> $GITHUB_OUTPUT - fi - - # Restore current state - git checkout ${{ github.sha }} - git stash pop - - - name: Upload Analysis Artifacts - uses: actions/upload-artifact@v3 - with: - name: understand-first-analysis - path: | - maps/ - tours/ - context-debt-dashboard.html - ci-artifacts/ - retention-days: 30 - - - name: Upload Code Map Delta - if: steps.diff.outputs.changes_detected == 'true' - uses: actions/upload-artifact@v3 - with: - name: code-map-delta - path: maps/delta.svg - retention-days: 30 - - - name: Upload Understanding Tour - if: steps.lens.outputs.lens_created == 'true' - uses: actions/upload-artifact@v3 - with: - name: understanding-tour - path: tours/pr-tour.md - retention-days: 30 - - - name: Upload Context Debt Dashboard - if: steps.lens.outputs.lens_created == 'true' - uses: actions/upload-artifact@v3 - with: - name: context-debt-dashboard - path: context-debt-dashboard.html - retention-days: 30 - - - name: Generate Enhanced PR Comment - if: github.event_name == 'pull_request' - uses: actions/github-script@v6 - with: - script: | - const fs = require('fs'); - const path = require('path'); - - // Read analysis results - let comment = '## 📊 Understand-First Analysis\n\n'; - - // Add scan results with enhanced formatting - comment += `### 🔍 Repository Scan\n`; - comment += `| Metric | Value |\n`; - comment += `|--------|-------|\n`; - comment += `| **Functions analyzed** | ${process.env.FUNCTIONS_COUNT || 0} |\n`; - comment += `| **Modules processed** | ${process.env.MODULES_COUNT || 0} |\n`; - comment += `| **Analysis time** | ${new Date().toISOString()} |\n\n`; - - // Add lens results if available - if (process.env.LENS_CREATED === 'true') { - comment += `### 🎯 Understanding Lens\n`; - comment += `| Metric | Value |\n`; - comment += `|--------|-------|\n`; - comment += `| **Functions in lens** | ${process.env.LENS_FUNCTIONS || 0} |\n`; - comment += `| **Analysis focus** | Changed files in this PR |\n`; - comment += `| **Coverage** | ${Math.round((process.env.LENS_FUNCTIONS || 0) / (process.env.FUNCTIONS_COUNT || 1) * 100)}% of total functions |\n\n`; - } - - // Add enhanced diff results if available - if (process.env.CHANGES_DETECTED === 'true') { - comment += `### 📈 Code Changes Analysis\n`; - comment += `| Change Type | Count | Impact |\n`; - comment += `|-------------|-------|--------|\n`; - comment += `| **Functions added** | ${process.env.ADDED_FUNCTIONS || 0} | ${(process.env.ADDED_FUNCTIONS || 0) > 0 ? '🟡 New functionality' : '✅ No additions'} |\n`; - comment += `| **Functions removed** | ${process.env.REMOVED_FUNCTIONS || 0} | ${(process.env.REMOVED_FUNCTIONS || 0) > 0 ? '🔴 Potential breaking change' : '✅ No removals'} |\n`; - comment += `| **Functions modified** | ${process.env.MODIFIED_FUNCTIONS || 0} | ${(process.env.MODIFIED_FUNCTIONS || 0) > 0 ? '🟡 Behavior changes' : '✅ No modifications'} |\n`; - comment += `| **Total changes** | ${(parseInt(process.env.ADDED_FUNCTIONS || 0) + parseInt(process.env.REMOVED_FUNCTIONS || 0) + parseInt(process.env.MODIFIED_FUNCTIONS || 0))} | ${(parseInt(process.env.ADDED_FUNCTIONS || 0) + parseInt(process.env.REMOVED_FUNCTIONS || 0) + parseInt(process.env.MODIFIED_FUNCTIONS || 0)) > 0 ? '📊 Changes detected' : '✅ No changes'} |\n\n`; - } - - // Add enhanced context debt insights - comment += `### 🧠 Context Debt Insights\n`; - comment += `| Metric | Value | Status |\n`; - comment += `|--------|-------|--------|\n`; - comment += `| **Understanding score** | ${process.env.UNDERSTANDING_SCORE || 'N/A'}/100 | ${(process.env.UNDERSTANDING_SCORE || 0) >= 80 ? '✅ Good' : (process.env.UNDERSTANDING_SCORE || 0) >= 60 ? '🟡 Fair' : '🔴 Needs improvement'} |\n`; - comment += `| **Complexity trend** | ${process.env.COMPLEXITY_TREND || 'Stable'} | ${process.env.COMPLEXITY_TREND === 'Low' ? '✅ Low' : process.env.COMPLEXITY_TREND === 'Medium' ? '🟡 Medium' : '🔴 High'} |\n`; - comment += `| **Side effects** | ${process.env.SIDE_EFFECTS_COUNT || 0} functions | ${(process.env.SIDE_EFFECTS_COUNT || 0) === 0 ? '✅ None' : '🟡 Present'} |\n`; - comment += `| **High complexity** | ${process.env.HIGH_COMPLEXITY_COUNT || 0} functions | ${(process.env.HIGH_COMPLEXITY_COUNT || 0) === 0 ? '✅ None' : '🔴 Present'} |\n\n`; - - // Add mini-map if available - if (fs.existsSync('maps/delta.svg')) { - comment += `### 🗺️ Code Map Delta\n`; - comment += `A visual representation of the changes in your codebase:\n\n`; - comment += `![Code Map Delta](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts/understand-first-analysis/maps/delta.svg)\n\n`; - } - - // Add understanding tour if available - if (fs.existsSync('tours/pr-tour.md')) { - comment += `### 📚 Understanding Tour\n`; - comment += `An interactive tour has been generated to help reviewers understand the changes:\n\n`; - comment += `**📖 [Download Understanding Tour](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts/understand-first-analysis/tours/pr-tour.md)**\n\n`; - } - - // Add context debt dashboard if available - if (fs.existsSync('context-debt-dashboard.html')) { - comment += `### 📊 Context Debt Dashboard\n`; - comment += `A visual dashboard showing code quality trends and understanding metrics:\n\n`; - comment += `**📈 [View Context Debt Dashboard](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts/understand-first-analysis/context-debt-dashboard.html)**\n\n`; - } - - // Add enhanced reviewer checklist - comment += `### ✅ Reviewer Checklist\n`; - comment += `- [ ] I have reviewed the understanding analysis above\n`; - comment += `- [ ] I understand the complexity and side effects of changes\n`; - comment += `- [ ] I have verified the changes align with the understanding tour\n`; - comment += `- [ ] I confirm this change maintains code quality standards\n`; - comment += `- [ ] I have checked for potential breaking changes in dependencies\n`; - comment += `- [ ] I have reviewed the code map delta for visual changes\n`; - comment += `- [ ] I have checked the context debt dashboard for quality trends\n\n`; - - // Add enhanced understanding requirements - comment += `### 📋 Understanding Requirements\n`; - if (process.env.HIGH_COMPLEXITY_COUNT > 0) { - comment += `- [ ] **Documentation needed**: ${process.env.HIGH_COMPLEXITY_COUNT} high complexity functions detected\n`; - } - if (process.env.SIDE_EFFECTS_COUNT > 0) { - comment += `- [ ] **Tests needed**: ${process.env.SIDE_EFFECTS_COUNT} functions with side effects should have comprehensive tests\n`; - } - if (process.env.REMOVED_FUNCTIONS > 0) { - comment += `- [ ] **Breaking change review**: ${process.env.REMOVED_FUNCTIONS} functions removed or significantly modified\n`; - } - if (process.env.UNDERSTANDING_SCORE < 60) { - comment += `- [ ] **Code quality improvement**: Understanding score is below 60, consider refactoring\n`; - } - comment += `\n`; - - // Add artifacts section - comment += `### 📦 Analysis Artifacts\n`; - comment += `The following artifacts have been generated and are available for download:\n\n`; - comment += `- **Repository Map**: Complete analysis of all functions\n`; - comment += `- **Understanding Lens**: Focused analysis of changed functions\n`; - comment += `- **Code Map Delta**: Visual representation of changes\n`; - comment += `- **Understanding Tour**: Step-by-step guide through changes\n`; - comment += `- **Context Debt Dashboard**: Quality metrics and trends\n\n`; - - comment += `**🔗 [Download All Artifacts](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts/understand-first-analysis)**\n\n`; - - comment += `---\n`; - comment += `*This analysis was generated by [Understand-First](https://github.com/SentinelOps-CI/understand-first) to help achieve TTU ≤ 10 minutes and TTFSC ≤ 1 day.*`; - - // Post comment - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: comment - }); - env: - FUNCTIONS_COUNT: ${{ steps.scan.outputs.functions_count }} - MODULES_COUNT: ${{ steps.scan.outputs.modules_count }} - LENS_CREATED: ${{ steps.lens.outputs.lens_created }} - LENS_FUNCTIONS: ${{ steps.lens.outputs.lens_functions }} - CHANGES_DETECTED: ${{ steps.diff.outputs.changes_detected }} - ADDED_FUNCTIONS: ${{ steps.diff.outputs.added_functions }} - REMOVED_FUNCTIONS: ${{ steps.diff.outputs.removed_functions }} - MODIFIED_FUNCTIONS: ${{ steps.diff.outputs.modified_functions }} - HIGH_COMPLEXITY_COUNT: ${{ steps.quality.outputs.high_complexity_count }} - SIDE_EFFECTS_COUNT: ${{ steps.quality.outputs.side_effects_count }} - UNDERSTANDING_SCORE: ${{ steps.quality.outputs.understanding_score }} - COMPLEXITY_TREND: ${{ steps.quality.outputs.complexity_trend }} - - - name: Analyze Code Quality - id: quality - if: steps.lens.outputs.lens_created == 'true' - run: | - echo "🔍 Analyzing code quality and understanding metrics..." - - # Calculate quality metrics - HIGH_COMPLEXITY=$(jq '[.functions | to_entries[] | select(.value.complexity > 10)] | length' maps/lens.json) - SIDE_EFFECTS=$(jq '[.functions | to_entries[] | select(.value.side_effects | length > 0)] | length' maps/lens.json) - - # Calculate average complexity - AVG_COMPLEXITY=$(jq '[.functions | to_entries[] | .value.complexity] | add / length' maps/lens.json) - - # Calculate understanding score (0-100) - TOTAL_FUNCTIONS=$(jq '.functions | length' maps/lens.json) - COMPLEXITY_PENALTY=$(echo "$HIGH_COMPLEXITY * 10" | bc) - SIDE_EFFECT_PENALTY=$(echo "$SIDE_EFFECTS * 5" | bc) - UNDERSTANDING_SCORE=$(echo "100 - $COMPLEXITY_PENALTY - $SIDE_EFFECT_PENALTY" | bc) - - # Determine complexity trend - if (( $(echo "$AVG_COMPLEXITY > 5" | bc -l) )); then - COMPLEXITY_TREND="High" - elif (( $(echo "$AVG_COMPLEXITY > 3" | bc -l) )); then - COMPLEXITY_TREND="Medium" - else - COMPLEXITY_TREND="Low" - fi - - echo "high_complexity_count=$HIGH_COMPLEXITY" >> $GITHUB_OUTPUT - echo "side_effects_count=$SIDE_EFFECTS" >> $GITHUB_OUTPUT - echo "understanding_score=$UNDERSTANDING_SCORE" >> $GITHUB_OUTPUT - echo "complexity_trend=$COMPLEXITY_TREND" >> $GITHUB_OUTPUT - echo "avg_complexity=$AVG_COMPLEXITY" >> $GITHUB_OUTPUT - - echo "📊 Quality Metrics:" - echo " - High complexity functions: $HIGH_COMPLEXITY" - echo " - Functions with side effects: $SIDE_EFFECTS" - echo " - Average complexity: $AVG_COMPLEXITY" - echo " - Understanding score: $UNDERSTANDING_SCORE/100" - echo " - Complexity trend: $COMPLEXITY_TREND" - - - name: Check Understanding Requirements - if: steps.lens.outputs.lens_created == 'true' - run: | - echo "🔍 Checking understanding requirements..." - - # Check if tour was generated - if [ ! -f "tours/pr-tour.md" ]; then - echo "❌ Understanding tour not generated" - exit 1 - fi - - # Check if analysis found high complexity functions - HIGH_COMPLEXITY=$(jq '[.functions | to_entries[] | select(.value.complexity > 10)] | length' maps/lens.json) - if [ "$HIGH_COMPLEXITY" -gt 0 ]; then - echo "⚠️ Warning: $HIGH_COMPLEXITY high complexity functions detected" - echo "Consider adding documentation or breaking down complex functions" - fi - - # Check for side effects - SIDE_EFFECTS=$(jq '[.functions | to_entries[] | select(.value.side_effects | length > 0)] | length' maps/lens.json) - if [ "$SIDE_EFFECTS" -gt 0 ]; then - echo "⚠️ Warning: $SIDE_EFFECTS functions with side effects detected" - echo "Ensure side effects are properly documented and tested" - fi - - echo "✅ Understanding requirements check passed" - - - name: Track TTU Metrics - run: | - echo "📊 Tracking TTU metrics..." - - # Record analysis completion time - ANALYSIS_TIME=$(date +%s) - echo "analysis_completed_at=$ANALYSIS_TIME" >> $GITHUB_OUTPUT - - # Record metrics for dashboard - echo "## Analysis Metrics" >> $GITHUB_STEP_SUMMARY - echo "- Functions analyzed: ${{ steps.scan.outputs.functions_count }}" >> $GITHUB_STEP_SUMMARY - echo "- Modules processed: ${{ steps.scan.outputs.modules_count }}" >> $GITHUB_STEP_SUMMARY - echo "- Understanding lens created: ${{ steps.lens.outputs.lens_created }}" >> $GITHUB_STEP_SUMMARY - echo "- Changes detected: ${{ steps.diff.outputs.changes_detected }}" >> $GITHUB_STEP_SUMMARY + echo "understand-first.yml is intentionally disabled." + echo "Use understand-first-pr-analysis.yml (scan + diff --old/--new)." + exit 0 diff --git a/.gitignore b/.gitignore index 383e9fd..026e8a5 100644 --- a/.gitignore +++ b/.gitignore @@ -141,6 +141,7 @@ metrics/ *.json !schemas/*.json !examples/**/*.json +!.bandit-baseline.json # IDE .vscode/ @@ -166,6 +167,8 @@ out.txt # Build artifacts build/ dist/ +dist_audit/ +dist_verify/ *.egg-info/ # Virtual environments diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 28b1aa7..84ceff6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,38 +4,54 @@ stages: - comment variables: - PIP_DISABLE_PIP_VERSION_CHECK: '1' + PIP_DISABLE_PIP_VERSION_CHECK: "1" +# Fail-closed build: scan/lens/trace/merge/tour must succeed. +# visual delta is optional nicety (continue via allow_failure on comment only). build_lens: stage: build image: python:3.10-slim script: - python -m pip install --upgrade pip - python -m pip install -e ".[dev,examples]" - - u scan examples/python_toy -o maps/repo.json - - u lens preset bug --map maps/repo.json -o maps/lens.json || true - - u trace module examples/app/hot_path.py run_hot_path -o traces/tour.json || true - - u lens merge-trace maps/lens.json traces/tour.json -o maps/lens_merged.json || true - - u tour maps/lens_merged.json -o tours/PR.md || true - - u visual delta maps/lens.json maps/lens_merged.json -o maps/delta.svg || true + - mkdir -p maps tours traces + - u scan . -o maps/repo.json + - u lens from-seeds --seed examples/app/hot_path.py --map maps/repo.json -o maps/lens.json + - nohup python examples/servers/http_server.py >/tmp/http.log 2>&1 & + - sleep 2 + - u trace module examples/app/hot_path.py run_hot_path -o traces/tour.json + - u lens merge-trace maps/lens.json traces/tour.json -o maps/lens_merged.json + - u tour maps/lens_merged.json -o tours/PR.md + - u visual delta maps/lens.json maps/lens_merged.json -o maps/delta.svg artifacts: paths: - tours/PR.md - maps/delta.svg - - docs/ + - maps/lens.json + - maps/lens_merged.json + - maps/repo.json + - traces/tour.json + expire_in: 1 week tour_gate: stage: tour image: python:3.10-slim + needs: ["build_lens"] script: - python -m pip install --upgrade pip - python -m pip install -e ".[dev,examples]" - - u tour_run maps/lens_merged.json -f fixtures || (echo "Tour failed" && exit 1) + - test -f maps/lens_merged.json || (echo "Missing maps/lens_merged.json from build_lens artifacts" && exit 1) + - u tour_run maps/lens_merged.json -f fixtures +# Soft-fail nicety only: posting a comment must not gate merge quality. +# allow_failure keeps the pipeline green if credentials/API fail; build_lens +# and tour_gate remain fail-closed. comment: stage: comment image: curlimages/curl:8.8.0 + allow_failure: true script: - - 'curl --header "PRIVATE-TOKEN: ${GITLAB_TOKEN}" --data-urlencode "body=See artifacts: tours/PR.md, maps/delta.svg" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/merge_requests/${CI_MERGE_REQUEST_IID}/notes" || true' + - echo "SOFT-FAIL (nicety): MR comment is optional; analysis already ran fail-closed." + - 'curl --header "PRIVATE-TOKEN: ${GITLAB_TOKEN}" --data-urlencode "body=See artifacts: tours/PR.md, maps/delta.svg" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/merge_requests/${CI_MERGE_REQUEST_IID}/notes"' rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' diff --git a/.understand-first.yml b/.understand-first.yml index fc75f65..71532cf 100644 --- a/.understand-first.yml +++ b/.understand-first.yml @@ -5,28 +5,3 @@ contracts_paths: [] glossary_path: docs/glossary.md metrics: enabled: true - weekly_reports: true - event_tracking: true -exclude_patterns: -- '**/__pycache__/**' -- '**/node_modules/**' -- '**/.git/**' -include_patterns: -- y -- y -- y -- y -analysis_options: - complexity_analysis: true - side_effects: true - dependencies: true -ci_integration: - enabled: true - platform: y - fail_on_issues: true - generate_reports: true -ide_integration: - enabled: true - type: y - gutter_annotations: true - quick_peek: true diff --git a/README.md b/README.md index 76a5c4e..fd9f74a 100644 --- a/README.md +++ b/README.md @@ -27,13 +27,13 @@ Large codebases reward familiarity over clarity. Docs drift, static graphs show | Capability | Understand-First | Static analysis | Doc generators | Review tools | |------------|------------------|-----------------|------------------|--------------| -| Runtime tracing | Yes | No | No | No | -| Guided tours | Yes | No | No | No | -| Living maps tied to the repo | Yes | Partial | Manual | No | -| PR / CI hooks for understanding artifacts | Yes | Rare | No | Partial | -| TTU / context-debt style metrics | Yes | No | No | No | -| Multi-language | Python today; more planned | Varies | Varies | Varies | -| IDE overlays | VS Code extension | Varies | No | Varies | +| Runtime tracing | Yes (opt-in `u trace`) | No | No | No | +| Guided tours | Yes (`u tour` from lenses) | No | No | No | +| Living maps tied to the repo | Yes (Python AST; JS/TS AST via Node when available, else regex fallback) | Partial | Manual | No | +| PR / CI hooks for understanding artifacts | Yes (when workflows enabled) | Rare | No | Partial | +| TTU / context-debt style metrics | Yes (map-derived; not invented scores) | No | No | No | +| Multi-language | Python (solid AST); JS/TS (Node TS AST + regex + barrels/`exports`); Go (`go/ast` + regex); Java (`javalang` + regex); Rust (`syn` AST when cargo present + regex fallback); C# (Roslyn AST when .NET SDK present + regex fallback); others not faked | Varies | Varies | Varies | +| IDE overlays | VS Code extension (map-backed) | Varies | No | Varies | | Contract extraction & checks | Yes | No | No | No | ### What you gain in practice @@ -47,8 +47,8 @@ Large codebases reward familiarity over clarity. Docs drift, static graphs show ## What it does -- **Maps** — Call and dependency graphs (Python-first; other language adapters are planned). -- **Invariants & effects** — Surfaces pre/post hints, suspected side effects, and contract-linked structure. +- **Maps** — Python: high-confidence AST call graphs with McCabe complexity and callers (import-gated / return-type factory edges when unique; ambiguous names omit rather than invent). JavaScript/TypeScript: Node TypeScript AST when available (`*-ast`, `ast-cyclomatic`, same-file + relative imports, re-export barrels, and nearest `package.json` `"exports"` subpaths); otherwise regex fallback. Go: `go/ast` when available; otherwise regex. Java: `javalang` when installed; otherwise regex. Rust: `syn` AST when cargo is available (`rust-ast`, `ast-cyclomatic`); otherwise `rust-best-effort` regex — **not** rustc typechecked/Python parity. C#: Roslyn AST when a .NET SDK is available (`csharp-ast`, `ast-cyclomatic`); otherwise `csharp-best-effort` regex — **not** typechecked/Python parity. Unsupported languages are counted and skipped (never scanned as Python or empty fake maps). +- **Invariants & effects** — Contract-linked structure and pre/post hints where configured; map `side_effects` tags are AST heuristics on Python, not proven purity (JS maps leave `side_effects` empty). - **Hot-path fixtures** — Small, runnable scaffolds tied to real execution. - **Reading plans** — Module-oriented summaries and questions before you edit. - **Proof-of-understanding hooks** — CI can require map deltas, invariants, or module READMEs where you configure it. @@ -140,7 +140,7 @@ u contracts lean-stubs contracts/contracts.yaml -o contracts/lean/ u contracts verify-lean contracts/contracts.yaml -l contracts/lean ``` -Explore `maps/`, `contracts/`, and `examples/` for generated artifacts. +`lean-stubs` / `verify-lean` are **scaffold / presence-only** (`Prop := True`, `by trivial`); they do not compile Lean. --- @@ -166,7 +166,7 @@ u tour maps/django_lens.json -o tours/django_tour.md ## Web demo -The [browser demo](web_demo/index.html) runs entirely on your machine: paste Python, inspect structure, explore tours and exports (JSON, Markdown, tour formats), and try complexity and call-graph views without installing the CLI. +The [browser demo](web_demo/index.html) is a **minimal** paste-box + graph sketch. It is **not** `u scan` / `u tour` / `u pack` parity — use the CLI for real maps (Python AST; JS/TS AST or regex fallback), complexity, callers, and exportable artifacts. --- @@ -194,8 +194,13 @@ Supported flavors include Django, FastAPI, React, Flask, microservices, Node, Go "file": "examples/python_toy/pkg/service.py", "calls": ["add", "maybe_log"], "callers": [], - "complexity": 3, - "side_effects": ["logging"] + "complexity": 1 + }, + "examples/python_toy/pkg/service:classify": { + "file": "examples/python_toy/pkg/service.py", + "calls": [], + "callers": [], + "complexity": 4 } } } @@ -252,11 +257,11 @@ u demo # contracts, services, trace, tour, dashboard; prints a local file UR - **Lenses & tours** — `u lens` (from-issue, from-seeds, merge-trace, preset, ingest-*, explain), `u tour`, `u tour_run` - **Tracing** — `u trace module`, `u trace errors` - **Boundaries** — `u boundaries scan` -- **Contracts** — `u contracts from-openapi`, `from-proto`, `compose`, `lean-stubs`, `verify-lean`, `stub-tests` +- **Contracts** — `u contracts from-openapi`, `from-proto`, `compose`, `lean-stubs` (scaffold), `verify-lean` (presence-only), `stub-tests` - **Visualization** — `u visual delta` -- **Packs** — `u pack create`, `u pack --publish` +- **Packs** — `u pack create`, `u pack publish` - **Glossary & dashboard** — `u glossary`, `u dashboard` -- **Health & config** — `u doctor`, `u ttu`, `u init`, `u tour_gate`, `u config_validate` +- **Health & config** — `u doctor`, `u ttu`, `u init`, `u tour_run` / `u tour_gate` (fixture + IDE progress), `u config_validate` See `u --help` for flags and subcommands. @@ -273,7 +278,8 @@ See `u --help` for flags and subcommands. | `ide/` | VS Code extension (maps, tours, error propagation) | | `examples/` | Django, FastAPI, Flask, React (Vite), microservices, toys | | `templates/` | Project-type starters for `u init` | -| `web_demo/` | Static interactive demo | +| `web_demo/` | Static browser-heuristic demo (not `u scan` parity) | +| `instrumentation/` | Standalone metrics stack; not wired into the `u` CLI | | `maps/` | Generated maps (JSON, DOT, Markdown, SVG) | | `tests/` | CLI and component tests | @@ -331,9 +337,9 @@ More detail: `docs/usage.md` and `docs/onboarding.md`. ## Roadmap -- Language adapters: TypeScript, Go, Java (see issues and `docs/` for direction). +- Language adapters: Kotlin, deeper crate-graph Rust resolve, richer C# `using`/project-reference edges, etc. Bare npm package-name resolve stays out of scope by design. - Map-delta visualizations and richer PR comment flows. -- Invariant DSL with optional Lean stubs. +- Invariant DSL with optional Lean scaffold stubs (presence-only; not compiled). - Deeper IDE integration beyond the current VS Code extension. --- diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 23246f7..151b497 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -2,34 +2,50 @@ image: python:3.10-slim pipelines: pull-requests: - '**': + "**": - step: - name: Build lens & tour + name: Build lens & tour (fail-closed) caches: - pip script: - python -m pip install --upgrade pip - python -m pip install -e ".[dev,examples]" - - u scan examples/python_toy -o maps/repo.json - - u lens preset bug --map maps/repo.json -o maps/lens.json || true - - u trace module examples/app/hot_path.py run_hot_path -o traces/tour.json || true - - u lens merge-trace maps/lens.json traces/tour.json -o maps/lens_merged.json || true - - u tour maps/lens_merged.json -o tours/PR.md || true - - u visual delta maps/lens.json maps/lens_merged.json -o maps/delta.svg || true + - mkdir -p maps tours traces + - u scan . -o maps/repo.json + - u lens from-seeds --seed examples/app/hot_path.py --map maps/repo.json -o maps/lens.json + - nohup python examples/servers/http_server.py >/tmp/http.log 2>&1 & + - sleep 2 + - u trace module examples/app/hot_path.py run_hot_path -o traces/tour.json + - u lens merge-trace maps/lens.json traces/tour.json -o maps/lens_merged.json + - u tour maps/lens_merged.json -o tours/PR.md + - u visual delta maps/lens.json maps/lens_merged.json -o maps/delta.svg artifacts: - tours/PR.md - maps/delta.svg + - maps/lens.json + - maps/lens_merged.json + - maps/repo.json - docs/** - step: name: Tour gate script: - python -m pip install --upgrade pip - python -m pip install -e ".[dev,examples]" + - test -f maps/lens_merged.json - u tour_run maps/lens_merged.json -f fixtures - step: - name: Comment MR + # Soft-fail nicety only: MR comment must not mask analysis failures. + # Explicitly labeled — exit 0 after logging; never silent curl soft-fail on + # critical scan/lens/tour paths (those are fail-closed above). + name: Comment MR (soft-fail nicety) script: - | + set +e curl -X POST -H "Content-Type: application/json" -u "$BITBUCKET_USERNAME:$BITBUCKET_APP_PASSWORD" \ -d '{"content": {"raw": "See artifacts: tours/PR.md, maps/delta.svg"}}' \ - "https://api.bitbucket.org/2.0/repositories/$BITBUCKET_REPO_FULL_NAME/pullrequests/$BITBUCKET_PR_ID/comments" || true + "https://api.bitbucket.org/2.0/repositories/$BITBUCKET_REPO_FULL_NAME/pullrequests/$BITBUCKET_PR_ID/comments" + status=$? + if [ "$status" -ne 0 ]; then + echo "SOFT-FAIL (nicety): MR comment post failed (exit $status). Analysis artifacts are unaffected." + fi + exit 0 diff --git a/cli/ucli/analyzers/__init__.py b/cli/ucli/analyzers/__init__.py new file mode 100644 index 0000000..2231584 --- /dev/null +++ b/cli/ucli/analyzers/__init__.py @@ -0,0 +1,21 @@ +"""Language analyzers and multi-language scan dispatch.""" + +from __future__ import annotations + +from ucli.analyzers.python_analyzer import build_python_map +from ucli.analyzers.registry import ( + build_repo_map, + format_unsupported_summary, + registered_adapters, + supported_extensions, + supported_languages, +) + +__all__ = [ + "build_python_map", + "build_repo_map", + "format_unsupported_summary", + "registered_adapters", + "supported_extensions", + "supported_languages", +] diff --git a/cli/ucli/analyzers/base.py b/cli/ucli/analyzers/base.py new file mode 100644 index 0000000..95d686e --- /dev/null +++ b/cli/ucli/analyzers/base.py @@ -0,0 +1,25 @@ +"""Language analyzer adapter contract. + +Adapters emit the same ``functions`` map shape as Python (qualified names, +``calls`` / ``callers`` / ``complexity`` / ``simple_name``) for the fields they +can honestly fill. Fidelity differs by language — see each adapter's ``note``. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Protocol, runtime_checkable + + +@runtime_checkable +class LanguageAdapter(Protocol): + """Protocol for per-language repository map builders.""" + + language: str + extensions: frozenset[str] + fidelity: str # e.g. "ast", "best-effort" + note: str + + def build_map(self, root: Path) -> dict[str, Any]: + """Return ``{"language": ..., "functions": {...}}`` for files under root.""" + ... diff --git a/cli/ucli/analyzers/map_meta.py b/cli/ucli/analyzers/map_meta.py new file mode 100644 index 0000000..d8ac74c --- /dev/null +++ b/cli/ucli/analyzers/map_meta.py @@ -0,0 +1,113 @@ +"""Shared map/lens metadata helpers for multi-language honesty (Wave 16).""" + +from __future__ import annotations + +from typing import Any + + +def map_fidelity_label(repo_map: dict[str, Any]) -> str: + """Return a short fidelity label for a map or lens carrying map metadata.""" + fidelity = repo_map.get("analyzer_fidelity") + if isinstance(fidelity, dict): + if not fidelity: + return "unknown" + vals = set(str(v) for v in fidelity.values()) + if vals == {"ast"}: + return "ast" + if "best-effort" in vals and "ast" in vals: + return "mixed" + if vals == {"best-effort"}: + return "best-effort" + return ",".join(sorted(vals)) + if isinstance(fidelity, str) and fidelity: + return fidelity + # Flat analyzer id on single-language maps + analyzer = str(repo_map.get("analyzer") or "") + if analyzer.endswith("-ast"): + return "ast" + if "best-effort" in analyzer: + return "best-effort" + # Infer from function analyzer tags + kinds = { + str(m.get("analyzer", "")) + for m in (repo_map.get("functions") or {}).values() + if isinstance(m, dict) + } + kinds.discard("") + if kinds and all(k.endswith("-ast") for k in kinds): + return "ast" + if kinds == {"python-ast"} or (kinds and all(k.startswith("python") for k in kinds)): + return "ast" + if kinds and all("best-effort" in k for k in kinds): + return "best-effort" + if kinds: + return "mixed" + lang = repo_map.get("language") + if lang in {"python"}: + return "ast" + if lang in {"javascript", "typescript", "go", "java", "rust", "csharp"}: + # Unknown path — do not claim AST. + return "best-effort" + if lang == "mixed": + return "mixed" + return "unknown" + + +def complexity_semantics_note(repo_map: dict[str, Any]) -> str: + label = map_fidelity_label(repo_map) + kind = repo_map.get("complexity_kind") + analyzer = str(repo_map.get("analyzer") or "") + if kind == "ast-cyclomatic" or ( + label == "ast" and analyzer.endswith("-ast") and analyzer != "python-ast" + ): + return ( + "Complexity values are ast-cyclomatic on JS/TS/Go/Java/Rust/C# AST " + "(decision points; see analyzer_note for formula) — not Python McCabe." + ) + if label == "ast": + return "Complexity values are McCabe cyclomatic (Python AST)." + if label == "best-effort": + return ( + "Complexity values are keyword-heuristic " + "(JS/TS/Go/Java/Rust/C# best-effort) — not McCabe AST." + ) + if label == "mixed": + return ( + "Complexity mixes McCabe / ast-cyclomatic (AST languages) and " + "keyword-heuristic (regex fallback); do not treat totals as uniform McCabe." + ) + return "Complexity semantics depend on the producing analyzer; check analyzer_fidelity." + + +def annotate_lens_from_map(lens: dict[str, Any], repo_map: dict[str, Any]) -> dict[str, Any]: + """Copy map language/fidelity metadata onto a lens for downstream honesty.""" + lens_meta = lens.setdefault("lens", {}) + for key in ( + "language", + "languages_analyzed", + "analyzer_fidelity", + "analyzer_notes", + "unsupported", + ): + if key in repo_map and key not in lens_meta: + lens_meta[key] = repo_map[key] + # Flat copies some consumers already look for on the document root. + if "language" in repo_map: + lens.setdefault("language", repo_map["language"]) + if "analyzer_fidelity" in repo_map: + lens.setdefault("analyzer_fidelity", repo_map["analyzer_fidelity"]) + lens_meta["complexity_note"] = complexity_semantics_note(repo_map) + lens_meta["fidelity"] = map_fidelity_label(repo_map) + return lens + + +def lens_has_python_files(lens: dict[str, Any]) -> bool: + for meta in (lens.get("functions") or {}).values(): + if isinstance(meta, dict) and str(meta.get("file", "")).endswith(".py"): + return True + return False + + +def lens_runtime_trace_supported(lens: dict[str, Any]) -> bool: + """Runtime ``u trace`` / ``tour_run`` fixtures are Python-only today.""" + return lens_has_python_files(lens) diff --git a/cli/ucli/analyzers/python_analyzer.py b/cli/ucli/analyzers/python_analyzer.py index 632e804..7873b60 100644 --- a/cli/ucli/analyzers/python_analyzer.py +++ b/cli/ucli/analyzers/python_analyzer.py @@ -1,34 +1,932 @@ from __future__ import annotations -import ast, json, os, pathlib, hashlib, sqlite3, time -from typing import Dict, Any, List, Optional, Tuple + +import ast +import hashlib +import json +import os +import pathlib +import sqlite3 +import time from multiprocessing import Pool, cpu_count +from typing import Any + +# --------------------------------------------------------------------------- +# Side-effect heuristics (high-confidence AST signals — NOT perfect analysis) +# +# Emitted tags (sorted unique list per function): +# writes_global – assignment to a name declared global/nonlocal +# writes_attribute – store to an attribute (e.g. self.x = ..., obj.y = ...) +# io_open – call to builtin open(...) +# io_print – call to print(...) +# io_write – .write / .writelines call +# io_read – .read / .readline / .readlines call +# network_call – local call whose callee looks like HTTP/socket I/O +# (Name in NETWORK_BARE, or Attribute on NETWORK_ROOTS) +# logging – logging.* or *.info/debug/warning/error/exception/critical +# +# False positives are acceptable when documented; absence of a tag never means +# "proven pure". Nested FunctionDef / AsyncFunctionDef / ClassDef bodies are +# attributed to the nested definition, not the enclosing function. +# --------------------------------------------------------------------------- + +_NETWORK_ROOTS = frozenset({"requests", "httpx", "urllib", "aiohttp", "socket", "http", "urllib3"}) +_NETWORK_BARE = frozenset( + {"urlopen", "urlretrieve", "urlfetch", "get", "post", "put", "patch", "delete"} +) +# Bare get/post are too ambiguous alone; only treat Attribute forms for those. +_NETWORK_BARE_STRICT = frozenset({"urlopen", "urlretrieve", "urlfetch"}) +_IO_WRITE_ATTRS = frozenset({"write", "writelines"}) +_IO_READ_ATTRS = frozenset({"read", "readline", "readlines"}) +_LOG_ATTRS = frozenset( + {"debug", "info", "warning", "warn", "error", "exception", "critical", "log"} +) def _is_code_file(path: pathlib.Path) -> bool: return path.suffix == ".py" and not any(part.startswith(".") for part in path.parts) -def _file_signature(path: pathlib.Path) -> Tuple[int, int, str]: +def _file_signature(path: pathlib.Path) -> tuple[int, int, str]: st = path.stat() h = hashlib.sha256(path.read_bytes()).hexdigest() return int(st.st_mtime), int(st.st_size), h -def _parse_file(file_path: pathlib.Path) -> Dict[str, Any]: +def _cyclomatic_complexity(func_node: ast.FunctionDef | ast.AsyncFunctionDef) -> int: + """McCabe-style cyclomatic complexity for a single function (nested defs excluded). + + Formula: + CC = 1 + + each If / IfExp / For / AsyncFor / While / Assert + + each ExceptHandler + + each extra Boolean operator in And/Or (len(values) - 1) + + each ``if`` filter in a comprehension / generator expression + + each ``match`` case (ast.Match) + + Nested FunctionDef / AsyncFunctionDef / ClassDef bodies are skipped so their + decisions are not attributed to the enclosing function. + """ + score = 1 + + class _CCVisitor(ast.NodeVisitor): + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: + # Nested function: do not descend. + return + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: + return + + def visit_ClassDef(self, node: ast.ClassDef) -> None: + return + + def visit_If(self, node: ast.If) -> None: + nonlocal score + score += 1 + self.generic_visit(node) + + def visit_IfExp(self, node: ast.IfExp) -> None: + nonlocal score + score += 1 + self.generic_visit(node) + + def visit_For(self, node: ast.For) -> None: + nonlocal score + score += 1 + self.generic_visit(node) + + def visit_AsyncFor(self, node: ast.AsyncFor) -> None: + nonlocal score + score += 1 + self.generic_visit(node) + + def visit_While(self, node: ast.While) -> None: + nonlocal score + score += 1 + self.generic_visit(node) + + def visit_Assert(self, node: ast.Assert) -> None: + nonlocal score + score += 1 + self.generic_visit(node) + + def visit_ExceptHandler(self, node: ast.ExceptHandler) -> None: + nonlocal score + score += 1 + self.generic_visit(node) + + def visit_BoolOp(self, node: ast.BoolOp) -> None: + nonlocal score + score += max(0, len(node.values) - 1) + self.generic_visit(node) + + def visit_comprehension(self, node: ast.comprehension) -> None: + nonlocal score + score += len(node.ifs) + self.generic_visit(node) + + def visit_Match(self, node: ast.AST) -> None: + # match/case (3.10+); typed as AST so pyright stays clean on 3.9. + nonlocal score + cases = getattr(node, "cases", None) or [] + score += len(cases) + self.generic_visit(node) + + visitor = _CCVisitor() + for stmt in func_node.body: + visitor.visit(stmt) + return score + + +def _side_effects_for_func(func_node: ast.FunctionDef | ast.AsyncFunctionDef) -> list[str]: + """Return sorted heuristic side-effect tags for one function body.""" + effects: set[str] = set() + external_names: set[str] = set() + + class _SEVisitor(ast.NodeVisitor): + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: + return + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: + return + + def visit_ClassDef(self, node: ast.ClassDef) -> None: + return + + def visit_Global(self, node: ast.Global) -> None: + external_names.update(node.names) + + def visit_Nonlocal(self, node: ast.Nonlocal) -> None: + external_names.update(node.names) + + def _mark_store(self, target: ast.AST) -> None: + if isinstance(target, ast.Name): + if target.id in external_names: + effects.add("writes_global") + elif isinstance(target, ast.Attribute): + effects.add("writes_attribute") + elif isinstance(target, (ast.Tuple, ast.List)): + for elt in target.elts: + self._mark_store(elt) + elif isinstance(target, ast.Starred): + self._mark_store(target.value) + + def visit_Assign(self, node: ast.Assign) -> None: + for t in node.targets: + self._mark_store(t) + self.generic_visit(node) + + def visit_AugAssign(self, node: ast.AugAssign) -> None: + self._mark_store(node.target) + self.generic_visit(node) + + def visit_AnnAssign(self, node: ast.AnnAssign) -> None: + if node.value is not None: + self._mark_store(node.target) + self.generic_visit(node) + + def visit_Call(self, node: ast.Call) -> None: + func = node.func + if isinstance(func, ast.Name): + name = func.id + if name == "open": + effects.add("io_open") + elif name == "print": + effects.add("io_print") + elif name in _NETWORK_BARE_STRICT: + effects.add("network_call") + elif isinstance(func, ast.Attribute): + attr = func.attr + if attr in _IO_WRITE_ATTRS: + effects.add("io_write") + elif attr in _IO_READ_ATTRS: + effects.add("io_read") + if attr in _LOG_ATTRS: + effects.add("logging") + # requests.get / httpx.post / socket.create_connection + root = func.value + if isinstance(root, ast.Name) and root.id in _NETWORK_ROOTS: + effects.add("network_call") + elif isinstance(root, ast.Attribute) and root.attr in _NETWORK_ROOTS: + effects.add("network_call") + self.generic_visit(node) + + visitor = _SEVisitor() + for stmt in func_node.body: + visitor.visit(stmt) + return sorted(effects) + + +def _annotation_present(node: ast.AST | None) -> bool: + return node is not None + + +def _func_type_hint_flags(node: ast.FunctionDef | ast.AsyncFunctionDef) -> dict[str, Any]: + """Simple AST type-hint / docstring gap signals (not full typing fidelity).""" + args = list(node.args.args) + list(node.args.kwonlyargs) + if node.args.vararg is not None: + args.append(node.args.vararg) + if node.args.kwarg is not None: + args.append(node.args.kwarg) + typed_params = sum(1 for a in args if _annotation_present(a.annotation)) + total_params = len(args) + has_return = _annotation_present(node.returns) + return { + "has_docstring": ast.get_docstring(node) is not None, + "has_return_annotation": has_return, + "has_type_hints": has_return or typed_params > 0, + "typed_params": typed_params, + "total_params": total_params, + # True when every named param (including self) has an annotation, or there are none. + "fully_typed_params": total_params == 0 or typed_params == total_params, + } + + +def _package_parts_for_file(file_path: pathlib.Path) -> list[str]: + """Infer package parts by walking up directories that contain ``__init__.py``. + + Namespace packages without ``__init__.py`` are not inferred (no invented edges). + """ + parts: list[str] = [] + try: + cur = file_path.resolve().parent + except OSError: + cur = file_path.parent + while True: + if not (cur / "__init__.py").is_file(): + break + parts.append(cur.name) + parent = cur.parent + if parent == cur: + break + cur = parent + parts.reverse() + return parts + + +def _resolve_relative_module(file_path: pathlib.Path, module: str | None, level: int) -> str | None: + """Resolve a relative ``from`` import to a dotted module path, or None if invalid. + + High-confidence only: refuses imports that climb above the inferred package root. + """ + if level <= 0: + return module + pkg_parts = _package_parts_for_file(file_path) + # level=1 stays in current package; level=2 goes to parent, etc. + up = level - 1 + if up > len(pkg_parts): + return None + base_parts = pkg_parts[: len(pkg_parts) - up] if up else list(pkg_parts) + if module: + mod_parts = [p for p in module.split(".") if p] + return ".".join(base_parts + mod_parts) if (base_parts or mod_parts) else None + return ".".join(base_parts) if base_parts else None + + +def _collect_import_aliases(tree: ast.AST, file_path: pathlib.Path) -> dict[str, str]: + """Map local alias -> module path fragment for high-confidence import resolution. + + Records ``from X import Y`` / ``from X import Y as Z`` where Y is a plain name. + Relative imports (``from .X import Y``, ``from .. import Y``) are resolved against + ``file_path``'s package parts when confidence is unambiguous. + Absolute ``import pkg.mod`` aliases are recorded as ``alias -> pkg.mod``. + + ``from``-import bindings use ``base:name`` so later resolution can distinguish + symbol imports (``from mod import fn``) from module imports + (``from pkg import mod`` → treat ``mod`` as submodule ``pkg.mod`` when a + scanned module uniquely matches). + """ + aliases: dict[str, str] = {} + + for node in getattr(tree, "body", []) or []: + if isinstance(node, ast.ImportFrom): + if node.level and node.level > 0: + base = _resolve_relative_module(file_path, node.module, node.level) + if not base: + continue + elif node.module is None: + continue + else: + base = node.module + for alias in node.names: + local = alias.asname or alias.name + if alias.name == "*": + continue + # from pkg.mod import func -> local name maps to pkg.mod:func candidate key + aliases[local] = f"{base}:{alias.name}" + elif isinstance(node, ast.Import): + for alias in node.names: + local = alias.asname or alias.name.split(".")[-1] + aliases[local] = alias.name + return aliases + + +def _attr_call_token(node: ast.Attribute) -> str: + """Serialize an attribute call target to a dotted token when rooted at a Name. + + ``mod.fn`` → ``mod.fn``; ``pkg.mod.fn`` → ``pkg.mod.fn``. + Non-Name roots (``foo().bar``, ``self.x.y``) return the terminal attr only so + we never invent a dotted chain we cannot resolve honestly. + """ + parts: list[str] = [] + cur: ast.AST = node + while isinstance(cur, ast.Attribute): + parts.append(cur.attr) + cur = cur.value + if isinstance(cur, ast.Name): + parts.append(cur.id) + parts.reverse() + return ".".join(parts) + return node.attr + + +# Names that are never treated as concrete class types for call resolution. +_NON_CLASS_TYPE_NAMES = frozenset( + { + "Any", + "object", + "None", + "True", + "False", + "Ellipsis", + "type", + "Callable", + "Literal", + "TypedDict", + "Protocol", + "TypeVar", + "Generic", + "ClassVar", + "Final", + "Self", + "str", + "int", + "float", + "bool", + "bytes", + "list", + "dict", + "set", + "tuple", + "frozenset", + } +) + + +def _simple_type_name(node: ast.AST | None) -> str | None: + """High-confidence concrete class name from an annotation or constructor call. + + Accepts bare ``Foo``, ``pkg.Foo``, and ``Optional[Foo]`` / ``Union[Foo, None]`` / + ``Foo | None`` when exactly one non-None concrete name remains. Rejects + multi-class unions, containers (``list[Foo]``), ``TypedDict`` / ``Protocol`` / + builtins / ``Any`` — never invents a type. + """ + if node is None: + return None + if isinstance(node, ast.Constant) and node.value is None: + return None + if isinstance(node, ast.Name): + if node.id in _NON_CLASS_TYPE_NAMES: + return None + return node.id + if isinstance(node, ast.Attribute): + # pkg.Foo → Foo (local Class.method resolve uses the terminal name) + if node.attr in _NON_CLASS_TYPE_NAMES: + return None + return node.attr + if isinstance(node, ast.Subscript): + # Optional[Foo], Union[Foo, None], typing.Optional[Foo] — single concrete + # class only. Skip TypedDict[...] / list[Foo] / dict[K, V] containers. + outer = _simple_type_name(node.value) + if outer in {"Optional", "Union"}: + return _unique_concrete_type(node.slice) + return None + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.BitOr): + return _unique_concrete_type(node) + if isinstance(node, ast.Tuple): + return _unique_concrete_type(node) + return None + + +def _unique_concrete_type(node: ast.AST | None) -> str | None: + """Return a class name only when exactly one concrete type appears.""" + names: list[str] = [] + + def walk(n: ast.AST | None) -> None: + if n is None: + return + if isinstance(n, ast.Constant) and n.value is None: + return + if isinstance(n, ast.BinOp) and isinstance(n.op, ast.BitOr): + walk(n.left) + walk(n.right) + return + if isinstance(n, ast.Tuple): + for elt in n.elts: + walk(elt) + return + name = _simple_type_name(n) + if name and name not in {"Optional", "Union"}: + names.append(name) + + walk(node) + uniq = list(dict.fromkeys(names)) + if len(uniq) == 1: + return uniq[0] + return None + + +def _looks_like_class_name(name: str) -> bool: + """PEP8 CapWords heuristic — snake_case calls are factories, not constructors.""" + return bool(name) and name[0].isupper() and name not in _NON_CLASS_TYPE_NAMES + + +def _constructor_type_name(node: ast.AST | None) -> str | None: + """Class name from ``Foo(...)`` / ``pkg.Foo(...)`` CapWords call expressions only. + + Snake_case ``make()`` is *not* treated as a constructor — those bind via + return-type propagation when the callee uniquely returns a concrete class. + """ + if not isinstance(node, ast.Call): + return None + name = _simple_type_name(node.func) + if name is None or not _looks_like_class_name(name): + return None + return name + + +def _factory_call_callee(node: ast.AST | None) -> str | None: + """Callee token for ``name = factory(...)`` when not a CapWords constructor.""" + if not isinstance(node, ast.Call): + return None + if _constructor_type_name(node) is not None: + return None + if isinstance(node.func, ast.Name): + return node.func.id + if isinstance(node.func, ast.Attribute): + return _attr_call_token(node.func) + return None + + +def _module_path_from_file(file_path: str) -> str: + """Best-effort dotted module path from a scanned file path (no package inventing).""" + posix = file_path.replace("\\", "/").removesuffix(".py") + if posix.endswith("/__init__"): + posix = posix[: -len("/__init__")] + return posix.replace("/", ".") + + +def _resolve_class_attr_in_file( + class_name: str, + attr: str, + origin_file: str, + by_short: dict[str, list[str]], + functions: dict[str, Any], +) -> list[str]: + """Unique ``Class.attr`` defined in ``origin_file``, else [].""" + local_key = f"{class_name}.{attr}" + candidates = [ + qn + for qn in dict.fromkeys(by_short.get(local_key, [])) + if qn.rsplit(":", 1)[-1] == local_key and functions[qn].get("file") == origin_file + ] + if len(candidates) == 1: + return candidates + return [] + + +def _canonicalize_type_via_imports( + name: str, + aliases: dict[str, str], +) -> tuple[str, str | None]: + """Map a local type name to ``(ClassName, origin_mod)`` via import aliases. + + ``from models import Foo as F`` → aliases ``F`` = ``models:Foo`` yields + ``(\"Foo\", \"models\")``. Bare names without a class import stay unchanged. + """ + binding = aliases.get(name) + if binding is not None and ":" in binding: + mod, imported = binding.rsplit(":", 1) + if _looks_like_class_name(imported): + return imported, mod or None + if _looks_like_class_name(name): + return name, mod or None + return name, None + + +def _module_path_for_attr_value( + value: ast.AST, + aliases: dict[str, str], +) -> str | None: + """Dotted module path for the root of ``models.Foo`` / ``pkg.mod.Foo``.""" + parts: list[str] = [] + cur: ast.AST = value + while isinstance(cur, ast.Attribute): + parts.append(cur.attr) + cur = cur.value + if not isinstance(cur, ast.Name): + return None + parts.append(cur.id) + parts.reverse() + root = parts[0] + rest = parts[1:] + binding = aliases.get(root) + if binding is None: + return None + if ":" not in binding: + return ".".join([binding, *rest]) if rest else binding + base, imported = binding.rsplit(":", 1) + mod = f"{base}.{imported}" if base else imported + return ".".join([mod, *rest]) if rest else mod + + +def _type_ref_with_origin( + node: ast.AST | None, + aliases: dict[str, str], +) -> tuple[str | None, str | None]: + """``(ClassName, origin_mod|None)`` from an annotation or constructor target. + + Origin is a dotted module when the type is import-aliased (``Foo`` / + ``F`` / ``models.Foo``). Optional/Union wrappers keep a single concrete + name; multi-class forms return ``(None, None)``. + """ + if node is None: + return None, None + if isinstance(node, ast.Call): + # CapWords constructor — origin from the callee expression. + if _constructor_type_name(node) is None: + return None, None + return _type_ref_with_origin(node.func, aliases) + if isinstance(node, ast.Name): + if node.id in _NON_CLASS_TYPE_NAMES: + return None, None + return _canonicalize_type_via_imports(node.id, aliases) + if isinstance(node, ast.Attribute): + if node.attr in _NON_CLASS_TYPE_NAMES: + return None, None + mod = _module_path_for_attr_value(node.value, aliases) + return node.attr, mod + if isinstance(node, ast.Subscript): + outer = _simple_type_name(node.value) + if outer in {"Optional", "Union"}: + inner = _unique_concrete_type(node.slice) + if not inner: + return None, None + # Prefer Attribute/Name origin inside the slice when unique. + return _type_ref_with_origin_for_name(node.slice, inner, aliases) + return None, None + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.BitOr): + inner = _unique_concrete_type(node) + if not inner: + return None, None + return _type_ref_with_origin_for_name(node, inner, aliases) + if isinstance(node, ast.Tuple): + inner = _unique_concrete_type(node) + if not inner: + return None, None + return _type_ref_with_origin_for_name(node, inner, aliases) + return None, None + + +def _type_ref_with_origin_for_name( + node: ast.AST | None, + target: str, + aliases: dict[str, str], +) -> tuple[str | None, str | None]: + """Locate ``target`` inside ``node`` and return canonical name + origin.""" + found: list[tuple[str, str | None]] = [] + + def walk(n: ast.AST | None) -> None: + if n is None: + return + if isinstance(n, ast.Constant) and n.value is None: + return + if isinstance(n, ast.BinOp) and isinstance(n.op, ast.BitOr): + walk(n.left) + walk(n.right) + return + if isinstance(n, ast.Tuple): + for elt in n.elts: + walk(elt) + return + if isinstance(n, ast.Name) and n.id == target: + found.append(_canonicalize_type_via_imports(n.id, aliases)) + return + if isinstance(n, ast.Attribute) and n.attr == target: + found.append((n.attr, _module_path_for_attr_value(n.value, aliases))) + return + # Optional[Foo] / Union[...] — descend into subscript slice only. + if isinstance(n, ast.Subscript): + walk(n.slice) + + walk(node) + if not found: + # Fall back to alias canonicalize of the simple name alone. + return _canonicalize_type_via_imports(target, aliases) + # All matches must agree on class + origin. + uniq = list(dict.fromkeys(found)) + if len(uniq) == 1: + return uniq[0] + # Class name agrees but origins differ → keep name, drop origin. + names = list(dict.fromkeys(c for c, _ in found)) + if len(names) == 1: + return names[0], None + return None, None + + +def _collect_return_factory_callees( + func_node: ast.FunctionDef | ast.AsyncFunctionDef, +) -> list[str] | None: + """Callee tokens when every non-None return is a factory result. + + Accepts ``return inner(...)`` and ``return x`` after ``x = inner(...)``. + CapWords constructors and unknown forms return ``None`` (not chainable here; + constructors are handled by ``_infer_return_type``). Empty eligible set → + ``None``. Used by map-level multi-hop chaining only. + """ + factory_assigns = _collect_factory_assigns(func_node) + found: list[str] = [] + + class ReturnFactoryVisitor(ast.NodeVisitor): + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: + if node is func_node: + self.generic_visit(node) + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: + if node is func_node: + self.generic_visit(node) + + def visit_ClassDef(self, node: ast.ClassDef) -> None: + return + + def visit_Return(self, node: ast.Return) -> None: + if node.value is None: + return + if isinstance(node.value, ast.Constant) and node.value.value is None: + return + factory = _factory_call_callee(node.value) + if factory is not None: + found.append(factory) + return + if isinstance(node.value, ast.Name) and node.value.id in factory_assigns: + found.append(factory_assigns[node.value.id]) + return + # Constructor / unknown / mixed — not a pure factory-return chain. + found.append("") + + ReturnFactoryVisitor().visit(func_node) + if not found or any(not t for t in found): + return None + return found + + +def _infer_return_type( + func_node: ast.FunctionDef | ast.AsyncFunctionDef, + import_aliases: dict[str, str] | None = None, +) -> tuple[str | None, str | None]: + """High-confidence ``(class_name, origin_mod|None)``, or ``(None, None)``. + + Prefer a unique return annotation. Else require every non-None ``return`` to + agree on one concrete class from ``return Class(...)`` or ``return alias`` + where ``alias`` is uniquely bound in-scope. Import aliases canonicalize + ``Foo as F`` / ``models.Foo`` and supply a dotted origin module for + cross-file ``obj.method`` gating. Multi-class / unknown forms → None. + + Factory-only returns (``return inner()``) are left unset here; map-level + ``_chain_factory_return_types`` may fill them when the callee's type is + unique within the hop cap. + """ + aliases = import_aliases or {} + ann_name, ann_mod = _type_ref_with_origin(func_node.returns, aliases) + if ann_name: + return ann_name, ann_mod + + # CapWords ctors + local aliases only (factory chains resolved later). + local = _collect_local_types(func_node) + found: list[tuple[str | None, str | None]] = [] + + class ReturnVisitor(ast.NodeVisitor): + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: + if node is func_node: + self.generic_visit(node) + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: + if node is func_node: + self.generic_visit(node) + + def visit_ClassDef(self, node: ast.ClassDef) -> None: + return + + def visit_Return(self, node: ast.Return) -> None: + if node.value is None: + return + if isinstance(node.value, ast.Constant) and node.value.value is None: + return + if isinstance(node.value, ast.Call) and _constructor_type_name(node.value): + found.append(_type_ref_with_origin(node.value, aliases)) + return + if isinstance(node.value, ast.Name): + bound = local.get(node.value.id) + if bound: + found.append(_canonicalize_type_via_imports(bound, aliases)) + return + if _looks_like_class_name(node.value.id): + found.append(_canonicalize_type_via_imports(node.value.id, aliases)) + return + # Unknown / non-concrete return form — poison uniqueness. + found.append((None, None)) + + ReturnVisitor().visit(func_node) + if any(t is None for t, _ in found): + return None, None + concrete = [(t, m) for t, m in found if t] + names = list(dict.fromkeys(t for t, _ in concrete)) + if len(names) != 1: + return None, None + mods = list(dict.fromkeys(m for _, m in concrete if m)) + # Agree on class; keep origin only when unanimous (or single known). + if len(mods) == 1: + return names[0], mods[0] + if len(mods) == 0: + return names[0], None + return names[0], None + + +def _collect_local_types( + func_node: ast.FunctionDef | ast.AsyncFunctionDef, +) -> dict[str, str]: + """Map local names → class names from annotations and constructors in scope. + + Only high-confidence bindings: annotated params/assigns and ``name = Class()``. + Later rebinding to another concrete type overwrites; ambiguous/unknown clears. + Factory calls (``name = make()``) are recorded separately for return-type + propagation at map-build time — they do not invent a class named ``make``. + """ + types: dict[str, str] = {} + + # Parameter annotations (skip self/cls without annotation). + for arg in [ + *func_node.args.posonlyargs, + *func_node.args.args, + *func_node.args.kwonlyargs, + ]: + if arg.arg in {"self", "cls"}: + continue + t = _simple_type_name(arg.annotation) + if t: + types[arg.arg] = t + + class BindingVisitor(ast.NodeVisitor): + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: + # Do not descend into nested defs — their bindings are their own. + if node is func_node: + self.generic_visit(node) + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: + if node is func_node: + self.generic_visit(node) + + def visit_ClassDef(self, node: ast.ClassDef) -> None: + return # nested class body is out of scope + + def visit_AnnAssign(self, node: ast.AnnAssign) -> None: + if isinstance(node.target, ast.Name): + ann = _simple_type_name(node.annotation) + ctor = _constructor_type_name(node.value) if node.value is not None else None + factory = _factory_call_callee(node.value) if node.value is not None else None + chosen = ctor or ann + if chosen: + types[node.target.id] = chosen + elif factory is not None: + # Pending return-type bind — clear stale annotation confidence. + types.pop(node.target.id, None) + elif node.target.id in types and ann is None and ctor is None: + # Explicit rebinding without a known type — drop confidence. + del types[node.target.id] + self.generic_visit(node) + + def visit_Assign(self, node: ast.Assign) -> None: + ctor = _constructor_type_name(node.value) + factory = _factory_call_callee(node.value) + for target in node.targets: + if isinstance(target, ast.Name): + if ctor: + types[target.id] = ctor + elif factory is not None: + types.pop(target.id, None) + elif target.id in types: + del types[target.id] + self.generic_visit(node) + + BindingVisitor().visit(func_node) + return types + + +def _collect_factory_assigns( + func_node: ast.FunctionDef | ast.AsyncFunctionDef, +) -> dict[str, str]: + """Map local names → callee tokens for ``name = factory(...)`` assigns. + + Last assignment wins. CapWords constructors are excluded (handled as types). + """ + assigns: dict[str, str] = {} + + class FactoryVisitor(ast.NodeVisitor): + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: + if node is func_node: + self.generic_visit(node) + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: + if node is func_node: + self.generic_visit(node) + + def visit_ClassDef(self, node: ast.ClassDef) -> None: + return + + def visit_AnnAssign(self, node: ast.AnnAssign) -> None: + if isinstance(node.target, ast.Name): + factory = _factory_call_callee(node.value) if node.value is not None else None + if factory: + assigns[node.target.id] = factory + else: + assigns.pop(node.target.id, None) + self.generic_visit(node) + + def visit_Assign(self, node: ast.Assign) -> None: + factory = _factory_call_callee(node.value) + for target in node.targets: + if isinstance(target, ast.Name): + if factory: + assigns[target.id] = factory + else: + assigns.pop(target.id, None) + self.generic_visit(node) + + FactoryVisitor().visit(func_node) + return assigns + + +def _local_func_name( + node: ast.FunctionDef | ast.AsyncFunctionDef, + class_stack: list[str], + *, + enclosing_func: str | None, + func_depth: int, +) -> str: + """Stable local name for map keys. + + * Class body methods: ``Class.method`` (nested classes: ``Outer.Inner.method``). + * Nested functions (inside any def): ``Enclosing.nested`` so they do not collide + with sibling methods or other nested defs. + * Top-level functions: bare ``def`` name (backward-compatible ``file:func``). + """ + if enclosing_func is not None and func_depth > 0: + return f"{enclosing_func}.{node.name}" + if class_stack and func_depth == 0: + return ".".join([*class_stack, node.name]) + return node.name + + +def _parse_file(file_path: pathlib.Path) -> dict[str, Any]: try: tree = ast.parse(file_path.read_text(encoding="utf-8")) except Exception: return {} - calls_by_func: Dict[str, List[str]] = {} - current_func: Optional[str] = None + calls_by_func: dict[str, list[str]] = {} + complexity_by_func: dict[str, int] = {} + side_effects_by_func: dict[str, list[str]] = {} + hint_flags_by_func: dict[str, dict[str, Any]] = {} + import_aliases = _collect_import_aliases(tree, file_path) + current_func: str | None = None + class_stack: list[str] = [] + func_depth = 0 class Visitor(ast.NodeVisitor): + def visit_ClassDef(self, node: ast.ClassDef) -> None: + class_stack.append(node.name) + self.generic_visit(node) + class_stack.pop() + def visit_FunctionDef(self, node: ast.FunctionDef): - nonlocal current_func - current_func = node.name + nonlocal current_func, func_depth + prev = current_func + current_func = _local_func_name( + node, + class_stack, + enclosing_func=prev, + func_depth=func_depth, + ) calls_by_func[current_func] = [] + complexity_by_func[current_func] = _cyclomatic_complexity(node) + side_effects_by_func[current_func] = _side_effects_for_func(node) + hint_flags_by_func[current_func] = _func_type_hint_flags(node) + func_depth += 1 self.generic_visit(node) - current_func = None + func_depth -= 1 + current_func = prev + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef): + # Treat async defs like sync defs for call-graph mapping + complexity. + self.visit_FunctionDef(node) # type: ignore[arg-type] def visit_Call(self, node: ast.Call): if current_func is not None: @@ -36,21 +934,754 @@ def visit_Call(self, node: ast.Call): if isinstance(node.func, ast.Name): callee = node.func.id elif isinstance(node.func, ast.Attribute): - callee = node.func.attr + # Name-rooted chains: mod.fn or pkg.mod.fn (resolved later). + callee = _attr_call_token(node.func) if callee: calls_by_func[current_func].append(callee) self.generic_visit(node) Visitor().visit(tree) + + # Second pass: local type bindings + enclosing class per function/method. + enclosing_by_func: dict[str, str | None] = {} + local_types_by_func: dict[str, dict[str, str]] = {} + factory_assigns_by_func: dict[str, dict[str, str]] = {} + return_type_by_func: dict[str, str | None] = {} + return_type_mod_by_func: dict[str, str | None] = {} + return_factory_callees_by_func: dict[str, list[str]] = {} + class_stack2: list[str] = [] + func_depth2 = 0 + current_func2: str | None = None + + class MetaVisitor(ast.NodeVisitor): + def visit_ClassDef(self, node: ast.ClassDef) -> None: + class_stack2.append(node.name) + self.generic_visit(node) + class_stack2.pop() + + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: + nonlocal current_func2, func_depth2 + prev = current_func2 + local = _local_func_name( + node, + class_stack2, + enclosing_func=prev, + func_depth=func_depth2, + ) + enclosing_by_func[local] = ( + ".".join(class_stack2) if class_stack2 and func_depth2 == 0 else None + ) + local_types_by_func[local] = _collect_local_types(node) + factory_assigns_by_func[local] = _collect_factory_assigns(node) + ret_name, ret_mod = _infer_return_type(node, import_aliases) + return_type_by_func[local] = ret_name + return_type_mod_by_func[local] = ret_mod + # Only record chain candidates when local inference did not settle. + if not ret_name: + callees = _collect_return_factory_callees(node) + if callees: + return_factory_callees_by_func[local] = callees + current_func2 = local + func_depth2 += 1 + self.generic_visit(node) + func_depth2 -= 1 + current_func2 = prev + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: + self.visit_FunctionDef(node) # type: ignore[arg-type] + + MetaVisitor().visit(tree) + out = {} for func, calls in calls_by_func.items(): - out[func] = { + flags = hint_flags_by_func.get(func, {}) + simple = func.rsplit(".", 1)[-1] + ret = return_type_by_func.get(func) + ret_mod = return_type_mod_by_func.get(func) + meta: dict[str, Any] = { "file": str(file_path.as_posix()), "calls": calls, + "callers": [], + "complexity": complexity_by_func.get(func, 1), + "side_effects": side_effects_by_func.get(func, []), + "has_docstring": bool(flags.get("has_docstring", False)), + "has_return_annotation": bool(flags.get("has_return_annotation", False)), + "has_type_hints": bool(flags.get("has_type_hints", False)), + "typed_params": int(flags.get("typed_params", 0)), + "total_params": int(flags.get("total_params", 0)), + "fully_typed_params": bool(flags.get("fully_typed_params", False)), + # Bare def/method name for consumers that still key on short names. + "simple_name": simple, + "analyzer": "python-ast", + "complexity_kind": "mccabe", + "language": "python", + # Local import map for cross-file resolution at map-build time. + "_import_aliases": import_aliases, + # Class body for methods (None for top-level / nested defs). + "_enclosing_class": enclosing_by_func.get(func), + # High-confidence local name → class bindings (annotations / ctors). + "_local_types": local_types_by_func.get(func) or {}, + # Module path gate for return-propagated bindings (filled later). + "_local_type_mods": {}, + # ``name = factory()`` pending binds for inter-procedural return types. + "_factory_assigns": factory_assigns_by_func.get(func) or {}, } + chain = return_factory_callees_by_func.get(func) + if chain: + meta["_return_factory_callees"] = chain + if ret: + meta["return_type"] = ret + meta["_return_type"] = ret + if ret_mod: + meta["_return_type_mod"] = ret_mod + out[func] = meta return out +def _match_module_path(file_path: str, mod_path: str) -> bool: + """True when ``file_path`` is the module file for dotted ``mod_path``. + + Matches ``pkg/mod.py`` for ``pkg.mod`` and ``pkg/__init__.py`` for ``pkg``. + """ + posix = file_path.replace("\\", "/") + mod_suffix = mod_path.replace(".", "/") + if not mod_suffix: + return False + if posix.endswith(mod_suffix + ".py"): + return True + if posix.endswith(mod_suffix + "/__init__.py"): + return True + return False + + +def _resolve_via_module_path( + mod_path: str, + func_name: str, + by_short: dict[str, list[str]], + functions: dict[str, Any], +) -> list[str]: + """Unique match of ``func_name`` under ``mod_path``, or empty if ambiguous/missing.""" + candidates = list(dict.fromkeys(by_short.get(func_name, []))) + if not candidates: + return [] + matched = [ + qn for qn in candidates if _match_module_path(functions[qn].get("file", ""), mod_path) + ] + # Prefer exact local-name match (``Class.method`` vs basename ``method``). + exact = [qn for qn in matched if qn.rsplit(":", 1)[-1] == func_name] + if len(exact) == 1: + return exact + if len(matched) == 1: + return matched + return [] + + +def _build_call_indexes( + functions: dict[str, Any], +) -> tuple[dict[str, list[str]], dict[str, dict[str, list[str]]]]: + """Build short-name and per-file short-name indexes in one O(n) pass. + + Keys after ``:`` may be ``func`` or ``Class.method``. Both the full local + name and the trailing method segment are indexed so ``self.method`` can + resolve to ``Class.method`` when unique. Duplicate basenames stay listed + (callers must apply uniqueness gates — never invent edges). + + ``by_file_short[file][short]`` avoids repeated full-map file filters during + attach/qualify (was O(candidates) scans per call site). + """ + by_short: dict[str, list[str]] = {} + by_file_short: dict[str, dict[str, list[str]]] = {} + for qn, meta in functions.items(): + short = qn.rsplit(":", 1)[-1] + file = meta.get("file", "") + by_short.setdefault(short, []).append(qn) + fmap = by_file_short.setdefault(file, {}) + fmap.setdefault(short, []).append(qn) + if "." in short: + base = short.rsplit(".", 1)[-1] + by_short.setdefault(base, []).append(qn) + fmap.setdefault(base, []).append(qn) + return by_short, by_file_short + + +def _index_by_short(functions: dict[str, Any]) -> dict[str, list[str]]: + """Index qnames by local name and method basename (thin wrapper).""" + by_short, _ = _build_call_indexes(functions) + return by_short + + +def _file_short_unique( + by_file_short: dict[str, dict[str, list[str]]], + caller_file: str, + key: str, +) -> list[str]: + """Unique same-file qnames for ``key``, else [].""" + local = list(dict.fromkeys(by_file_short.get(caller_file, {}).get(key, []))) + if len(local) == 1: + return local + return [] + + +def _resolve_typed_attr( + class_name: str, + attr: str, + by_short: dict[str, list[str]], + functions: dict[str, Any], + caller_file: str, + by_file_short: dict[str, dict[str, list[str]]] | None = None, +) -> list[str]: + """Resolve ``Class.attr`` when type is known — uniqueness-gated, no fan-out. + + Prefers same-file exact local name ``Class.attr``. Falls back to a single + global exact match. Never returns multiple candidates. + """ + local_key = f"{class_name}.{attr}" + if by_file_short is not None: + same_file = _file_short_unique(by_file_short, caller_file, local_key) + if same_file: + return same_file + candidates = list(dict.fromkeys(by_short.get(local_key, []))) + # Also accept basename index entries that are exactly Class.attr + for qn in by_short.get(attr, []): + if qn.rsplit(":", 1)[-1] == local_key and qn not in candidates: + candidates.append(qn) + exact = [qn for qn in candidates if qn.rsplit(":", 1)[-1] == local_key] + if by_file_short is None: + same_file = [qn for qn in exact if functions[qn].get("file") == caller_file] + if len(same_file) == 1: + return same_file + if len(exact) == 1: + return exact + return [] + + +def _resolve_imported_class_attr( + mod_path: str, + class_name: str, + attr: str, + by_short: dict[str, list[str]], + functions: dict[str, Any], +) -> list[str]: + """Resolve ``Class.attr`` under imported module ``mod_path`` when unique. + + Used for ``from mod import Foo`` then ``Foo.method`` / typed ``obj.method`` + where ``obj`` is annotated or constructed as ``Foo``. Ambiguous or missing + matches return [] — never invent edges. + """ + local_key = f"{class_name}.{attr}" + candidates = [ + qn + for qn in dict.fromkeys(by_short.get(local_key, [])) + if qn.rsplit(":", 1)[-1] == local_key + and _match_module_path(functions[qn].get("file", ""), mod_path) + ] + if len(candidates) == 1: + return candidates + return [] + + +def _resolve_class_attr_via_aliases( + class_name: str, + attr: str, + aliases: dict[str, str], + by_short: dict[str, list[str]], + functions: dict[str, Any], + caller_file: str, + *, + origin_mod: str | None = None, +) -> list[str]: + """Typed / imported-class resolve with import-alias gating when available. + + If ``class_name`` maps to ``from mod import Class`` / ``as`` alias + (``mod:Class``), prefer the module-gated ``Class.attr`` match — this + uniquifies cross-file even when another scanned file defines a same-named + class. When the import is present but no unique method exists, omit (do not + fall back to a different file's class). + + ``origin_mod`` may be: + * a defining **file path** (return-type propagation from a factory file), or + * a dotted **module path** (imported return type / ``models.Foo`` annotation). + + File origins prefer ``Class.attr`` in that file, then the file's module path, + then unique global. Module origins are import-gated only — no ambiguous + global fallthrough. + """ + binding = aliases.get(class_name) + if binding is not None and ":" in binding: + base, imported = binding.rsplit(":", 1) + via = _resolve_imported_class_attr(base, imported, attr, by_short, functions) + if via: + return via + # Import known but no unique Class.attr under that module — omit. + return [] + if origin_mod: + # Dotted module path (no path separators / .py suffix). + is_file = origin_mod.endswith(".py") or "/" in origin_mod or "\\" in origin_mod + if not is_file: + via_mod = _resolve_imported_class_attr( + origin_mod, class_name, attr, by_short, functions + ) + if via_mod: + return via_mod + # Known imported-return origin but no unique match — refuse. + return [] + # origin_mod holds the factory's file path (not a dotted module). + via_file = _resolve_class_attr_in_file(class_name, attr, origin_mod, by_short, functions) + if via_file: + return via_file + mod = _module_path_from_file(origin_mod) + if mod: + via_mod = _resolve_imported_class_attr(mod, class_name, attr, by_short, functions) + if via_mod: + return via_mod + return _resolve_typed_attr(class_name, attr, by_short, functions, caller_file) + return _resolve_typed_attr(class_name, attr, by_short, functions, caller_file) + + +def _resolve_attr_call( + dotted: str, + aliases: dict[str, str], + by_short: dict[str, list[str]], + functions: dict[str, Any], + caller_file: str, + *, + enclosing_class: str | None = None, + local_types: dict[str, str] | None = None, + local_type_mods: dict[str, str] | None = None, + by_file_short: dict[str, dict[str, list[str]]] | None = None, +) -> list[str]: + """Resolve ``mod.fn`` / ``self.method`` / typed ``obj.method`` / ``pkg.mod.fn``. + + Policy: + * ``root.attr`` (one dot): + - ``import`` alias (no ``:``): unique module-path resolve. + - ``from pkg import mod`` (``pkg:mod``): if ``pkg.mod`` uniquely hosts + ``attr``, use it (module-from-import). + - ``from mod import Foo`` (class): ``Foo.method`` via module-gated + ``Class.method`` when unique. + - ``self`` with known enclosing class: ``Class.attr`` uniqueness gate. + - Root with high-confidence local type (annotation/ctor/return-prop): + typed resolve (import-alias or factory-module gated when known). + - Untyped ``obj.method`` / unbound ``self``: **refuse** (do not guess + via same-file unique basename — that invented edges). + * ``pkg.mod.fn`` (two dots), high confidence only: + - ``pkg`` is an ``import`` alias → resolve ``pkg.mod`` + ``fn``. + - Else omit (do not invent nested edges). + * Longer chains: omit. + * Never fan out to all global short-name matches for attribute forms. + """ + parts = dotted.split(".") + if len(parts) == 2: + root, attr = parts + binding = aliases.get(root) + if binding is not None and ":" not in binding: + return _resolve_via_module_path(binding, attr, by_short, functions) + if binding is not None and ":" in binding: + # from pkg import mod → binding "pkg:mod"; try submodule pkg.mod. + base, imported = binding.rsplit(":", 1) + submodule = f"{base}.{imported}" if base else imported + via_mod = _resolve_via_module_path(submodule, attr, by_short, functions) + if via_mod: + return via_mod + # from mod import Foo → Foo.method (imported class, not a submodule). + via_cls = _resolve_imported_class_attr(base, imported, attr, by_short, functions) + if via_cls: + return via_cls + # Imported symbol is neither a module nor a scanned class method. + return [] + if root == "self": + # Honesty: only Class.method when enclosing class is known. + if not enclosing_class: + return [] + return _resolve_typed_attr( + enclosing_class, + attr, + by_short, + functions, + caller_file, + by_file_short=by_file_short, + ) + local_types = local_types or {} + local_type_mods = local_type_mods or {} + if root in local_types: + typed = _resolve_class_attr_via_aliases( + local_types[root], + attr, + aliases, + by_short, + functions, + caller_file, + origin_mod=local_type_mods.get(root), + ) + if typed: + return typed + return [] + # Untyped obj.method — refuse rather than same-file basename guess. + return [] + if len(parts) == 3: + root, mid, attr = parts + binding = aliases.get(root) + if binding is not None and ":" not in binding: + return _resolve_via_module_path(f"{binding}.{mid}", attr, by_short, functions) + return [] + return [] + + +def _resolve_bare_or_dotted_callee( + callee_token: str, + aliases: dict[str, str], + by_short: dict[str, list[str]], + functions: dict[str, Any], + caller_file: str, + caller_qn: str, + by_file_short: dict[str, dict[str, list[str]]] | None = None, +) -> list[str]: + """Unique callee qname for a bare or dotted call token (same policy as callers).""" + if "." in callee_token: + return _resolve_attr_call( + callee_token, + aliases, + by_short, + functions, + caller_file, + by_file_short=by_file_short, + ) + if by_file_short is not None: + same_file = list(dict.fromkeys(by_file_short.get(caller_file, {}).get(callee_token, []))) + else: + candidates = list(dict.fromkeys(by_short.get(callee_token, []))) + same_file = [c for c in candidates if functions[c].get("file") == caller_file] + if same_file: + caller_local = caller_qn.rsplit(":", 1)[-1] + nested = [c for c in same_file if c.rsplit(":", 1)[-1] == f"{caller_local}.{callee_token}"] + if len(nested) == 1: + return nested + exact = [c for c in same_file if c.rsplit(":", 1)[-1] == callee_token] + if len(exact) == 1: + return exact + if exact: + return [] + return same_file if len(same_file) == 1 else [] + candidates = list(dict.fromkeys(by_short.get(callee_token, []))) + if not candidates: + return [] + imported = _resolve_via_import(callee_token, aliases, by_short, functions) + if imported: + return imported + if len(candidates) == 1: + return candidates + return [] + + +def _propagate_return_types( + functions: dict[str, Any], + *, + by_short: dict[str, list[str]] | None = None, + by_file_short: dict[str, dict[str, list[str]]] | None = None, +) -> None: + """Bind ``obj = factory()`` locals from high-confidence callee return types. + + Only when the factory uniquely resolves and exposes a single concrete + ``_return_type``. Origin gating for later ``obj.method`` resolve prefers + ``_return_type_mod`` (imported / ``models.Foo``) when present; otherwise + the factory's defining file. Ambiguous factories / multi-return types + leave the bind unset. + """ + if by_short is None or by_file_short is None: + by_short, by_file_short = _build_call_indexes(functions) + for caller_qn, meta in functions.items(): + factories: dict[str, str] = meta.get("_factory_assigns") or {} + if not factories: + continue + aliases: dict[str, str] = meta.get("_import_aliases") or {} + caller_file = meta.get("file", "") + local_types: dict[str, str] = meta.setdefault("_local_types", {}) + local_mods: dict[str, str] = meta.setdefault("_local_type_mods", {}) + for local_name, callee_token in factories.items(): + if local_name in local_types: + # Annotation / ctor already won — do not overwrite. + continue + chosen = _resolve_bare_or_dotted_callee( + callee_token, + aliases, + by_short, + functions, + caller_file, + caller_qn, + by_file_short=by_file_short, + ) + if len(chosen) != 1: + continue + callee_meta = functions[chosen[0]] + ret = callee_meta.get("_return_type") or callee_meta.get("return_type") + if not ret or not isinstance(ret, str): + continue + local_types[local_name] = ret + ret_mod = callee_meta.get("_return_type_mod") + if isinstance(ret_mod, str) and ret_mod: + # Dotted module from imported return type — import-gated resolve. + local_mods[local_name] = ret_mod + else: + # Store defining file for Class.attr gating (absolute paths are OK). + local_mods[local_name] = callee_meta.get("file", "") + + +_MAX_RETURN_TYPE_HOPS = 3 + + +def _chain_factory_return_types( + functions: dict[str, Any], + *, + max_hops: int = _MAX_RETURN_TYPE_HOPS, + by_short: dict[str, list[str]] | None = None, + by_file_short: dict[str, dict[str, list[str]]] | None = None, +) -> None: + """Propagate return types through ``return inner()`` factory chains. + + Only when every non-None return is a uniquely resolvable factory whose + concrete ``_return_type`` agrees. Cap at ``max_hops`` rounds so + ``outer → mid → inner → Foo`` (3 hops) can settle; longer / cyclic / + ambiguous chains stay unset. Runs before ``_propagate_return_types``. + """ + if by_short is None or by_file_short is None: + by_short, by_file_short = _build_call_indexes(functions) + for _ in range(max(0, max_hops)): + progressed = False + for caller_qn, meta in functions.items(): + if meta.get("_return_type") or meta.get("return_type"): + continue + tokens: list[str] = meta.get("_return_factory_callees") or [] + if not tokens: + continue + aliases: dict[str, str] = meta.get("_import_aliases") or {} + caller_file = meta.get("file", "") + resolved: list[tuple[str, str | None]] = [] + incomplete = False + refuse = False + for tok in tokens: + chosen = _resolve_bare_or_dotted_callee( + tok, + aliases, + by_short, + functions, + caller_file, + caller_qn, + by_file_short=by_file_short, + ) + if len(chosen) != 1: + refuse = True + break + cal_qn = chosen[0] + if cal_qn == caller_qn: + refuse = True + break + cal_meta = functions[cal_qn] + ret = cal_meta.get("_return_type") or cal_meta.get("return_type") + if not ret or not isinstance(ret, str): + incomplete = True + break + ret_mod = cal_meta.get("_return_type_mod") + if isinstance(ret_mod, str) and ret_mod: + origin: str | None = ret_mod + else: + # Prefer callee file so later obj.method gating stays local. + origin = cal_meta.get("file") or None + resolved.append((ret, origin)) + if refuse or incomplete or not resolved: + if refuse: + # Ambiguous / self-cycle — drop chain candidate permanently. + meta.pop("_return_factory_callees", None) + continue + names = list(dict.fromkeys(r for r, _ in resolved)) + if len(names) != 1: + meta.pop("_return_factory_callees", None) + continue + mods = list(dict.fromkeys(m for _, m in resolved if m)) + meta["_return_type"] = names[0] + meta["return_type"] = names[0] + if len(mods) == 1: + meta["_return_type_mod"] = mods[0] + else: + meta.pop("_return_type_mod", None) + meta.pop("_return_factory_callees", None) + progressed = True + if not progressed: + break + + +def _attach_callers( + functions: dict[str, Any], + *, + by_short: dict[str, list[str]] | None = None, + by_file_short: dict[str, dict[str, list[str]]] | None = None, +) -> None: + """Populate reverse call edges (callers) from ``calls`` lists. + + Resolution policy (reduces cross-file false positives): + 1. Dotted forms: high-confidence module / module-from-import / typed / + enclosing-class ``self`` — never untyped same-file basename guess. + 2. Bare names: prefer same-file callees when any exist. + 3. Else, if import alias maps the name to ``module:func`` and exactly one + global qn matches that module path + func (high confidence), use it. + 4. Else, if exactly one global candidate shares the short name, use it. + 5. Else (ambiguous cross-file), skip the edge — do not fan out to all. + """ + if by_short is None or by_file_short is None: + by_short, by_file_short = _build_call_indexes(functions) + + for meta in functions.values(): + meta.setdefault("callers", []) + + for caller_qn, meta in functions.items(): + caller_file = meta.get("file", "") + aliases: dict[str, str] = meta.get("_import_aliases") or {} + enclosing = meta.get("_enclosing_class") + local_types: dict[str, str] = meta.get("_local_types") or {} + local_type_mods: dict[str, str] = meta.get("_local_type_mods") or {} + for callee_token in meta.get("calls", []): + if "." in callee_token: + chosen = _resolve_attr_call( + callee_token, + aliases, + by_short, + functions, + caller_file, + enclosing_class=enclosing, + local_types=local_types, + local_type_mods=local_type_mods, + by_file_short=by_file_short, + ) + if not chosen: + continue + else: + candidates = list(dict.fromkeys(by_short.get(callee_token, []))) + if not candidates: + continue + same_file = list( + dict.fromkeys(by_file_short.get(caller_file, {}).get(callee_token, [])) + ) + if same_file: + caller_local = caller_qn.rsplit(":", 1)[-1] + # Nested def under this caller: Foo.run -> helper => Foo.run.helper + nested = [ + c + for c in same_file + if c.rsplit(":", 1)[-1] == f"{caller_local}.{callee_token}" + ] + if len(nested) == 1: + chosen = nested + else: + # Prefer exact local-name key match when basename indexing + # pulled in Class.method alongside a top-level def. + exact = [c for c in same_file if c.rsplit(":", 1)[-1] == callee_token] + if exact: + chosen = exact + elif len(same_file) == 1: + chosen = same_file + else: + # Ambiguous Class.method basenames — omit. + continue + else: + chosen = _resolve_via_import(callee_token, aliases, by_short, functions) + if not chosen: + if len(candidates) == 1: + chosen = candidates + else: + # Ambiguous across files — omit rather than invent edges. + continue + for callee_qn in chosen: + if callee_qn != caller_qn: + functions[callee_qn]["callers"].append(caller_qn) + + for meta in functions.values(): + meta["callers"] = sorted(set(meta["callers"])) + + +def _resolve_via_import( + callee_short: str, + aliases: dict[str, str], + by_short: dict[str, list[str]], + functions: dict[str, Any], +) -> list[str]: + """High-confidence cross-file resolve using ``from mod import name`` aliases.""" + target = aliases.get(callee_short) + if not target or ":" not in target: + return [] + mod_path, func_name = target.rsplit(":", 1) + return _resolve_via_module_path(mod_path, func_name, by_short, functions) + + +def _qualify_calls( + functions: dict[str, Any], + *, + by_short: dict[str, list[str]] | None = None, + by_file_short: dict[str, dict[str, list[str]]] | None = None, +) -> None: + """Rewrite ``calls`` to file-local qualified names when unambiguous. + + Same-file unique callees become ``rel/path:func`` (or ``rel/path:Class.method``). + Unresolved / ambiguous names stay as bare or dotted tokens. Import-resolved + cross-file edges also get the matched qn when unique. Strips internal + ``_import_aliases``, ``_enclosing_class``, ``_local_types``, ``_local_type_mods``, + ``_factory_assigns``, ``_return_factory_callees``, ``_return_type``, + ``_return_type_mod``. + """ + if by_short is None or by_file_short is None: + by_short, by_file_short = _build_call_indexes(functions) + + for _caller_qn, meta in functions.items(): + aliases: dict[str, str] = meta.pop("_import_aliases", None) or {} + enclosing = meta.pop("_enclosing_class", None) + local_types: dict[str, str] = meta.pop("_local_types", None) or {} + local_type_mods: dict[str, str] = meta.pop("_local_type_mods", None) or {} + meta.pop("_factory_assigns", None) + meta.pop("_return_factory_callees", None) + meta.pop("_return_type", None) + meta.pop("_return_type_mod", None) + caller_file = meta.get("file", "") + qualified: list[str] = [] + for callee_token in meta.get("calls", []): + if "." in callee_token: + resolved = _resolve_attr_call( + callee_token, + aliases, + by_short, + functions, + caller_file, + enclosing_class=enclosing, + local_types=local_types, + local_type_mods=local_type_mods, + by_file_short=by_file_short, + ) + if len(resolved) == 1: + qualified.append(resolved[0]) + else: + qualified.append(callee_token) + continue + local_map = by_file_short.get(caller_file, {}) + local = list(dict.fromkeys(local_map.get(callee_token, []))) + caller_local = _caller_qn.rsplit(":", 1)[-1] + nested = [c for c in local if c.rsplit(":", 1)[-1] == f"{caller_local}.{callee_token}"] + exact = [c for c in local if c.rsplit(":", 1)[-1] == callee_token] + if len(nested) == 1: + pick = nested + elif len(exact) == 1: + pick = exact + elif len(local) == 1: + pick = local + else: + pick = [] + if len(pick) == 1: + qualified.append(pick[0]) + continue + imported = _resolve_via_import(callee_token, aliases, by_short, functions) + if len(imported) == 1: + qualified.append(imported[0]) + continue + qualified.append(callee_token) + meta["calls"] = qualified + + def _open_cache(cache_path: pathlib.Path): conn = sqlite3.connect(str(cache_path)) conn.execute( @@ -62,11 +1693,11 @@ def _open_cache(cache_path: pathlib.Path): def build_python_map( root: pathlib.Path, use_cache: bool = False, - processes: Optional[int] = None, - cache_path: Optional[pathlib.Path] = None, -) -> Dict[str, Any]: - functions: Dict[str, Any] = {} - files: List[pathlib.Path] = [] + processes: int | None = None, + cache_path: pathlib.Path | None = None, +) -> dict[str, Any]: + functions: dict[str, Any] = {} + files: list[pathlib.Path] = [] for p, _, files_in_dir in os.walk(root): for name in files_in_dir: path = pathlib.Path(p) / name @@ -88,7 +1719,7 @@ def qn(file_path: pathlib.Path, func_name: str) -> str: ): cache[path] = (mtime, size, sha, data) - work: List[pathlib.Path] = [] + work: list[pathlib.Path] = [] for file_path in files: if cache_conn is not None: mtime, size, sha = _file_signature(file_path) @@ -101,7 +1732,7 @@ def qn(file_path: pathlib.Path, func_name: str) -> str: continue work.append(file_path) - start = time.time() + time.time() results = [] if work: procs = max(1, processes or min(4, cpu_count())) @@ -125,4 +1756,19 @@ def qn(file_path: pathlib.Path, func_name: str) -> str: if cache_conn is not None: cache_conn.commit() - return {"language": "python", "functions": functions} + by_short, by_file_short = _build_call_indexes(functions) + _chain_factory_return_types(functions, by_short=by_short, by_file_short=by_file_short) + _propagate_return_types(functions, by_short=by_short, by_file_short=by_file_short) + _attach_callers(functions, by_short=by_short, by_file_short=by_file_short) + _qualify_calls(functions, by_short=by_short, by_file_short=by_file_short) + return { + "language": "python", + "functions": functions, + "analyzer": "python-ast", + "analyzer_fidelity": "ast", + "complexity_kind": "mccabe", + "analyzer_note": ( + "Python AST map with McCabe complexity; call edges are high-confidence " + "only (ambiguous names omit rather than invent)." + ), + } diff --git a/cli/ucli/analyzers/registry.py b/cli/ucli/analyzers/registry.py new file mode 100644 index 0000000..6fb6b7d --- /dev/null +++ b/cli/ucli/analyzers/registry.py @@ -0,0 +1,288 @@ +"""Analyzer registry and multi-language ``build_repo_map`` dispatch.""" + +from __future__ import annotations + +import os +import pathlib +from collections.abc import Iterable, Sequence +from typing import Any + +from ucli.analyzers.base import LanguageAdapter +from ucli.analyzers.python_analyzer import build_python_map + +# Extensions that look like source but have no adapter yet. +_UNSUPPORTED_SOURCE_EXTS = frozenset( + { + ".rb", + ".php", + ".kt", + ".swift", + ".c", + ".cc", + ".cpp", + ".cxx", + ".h", + ".hpp", + ".hh", + ".scala", + ".m", + ".mm", + ".lua", + ".r", + ".R", + ".jl", + ".zig", + ".dart", + ".vue", + ".svelte", + } +) + +_SKIP_DIR_NAMES = frozenset( + { + ".git", + ".hg", + ".svn", + "node_modules", + "__pycache__", + ".venv", + "venv", + "dist", + "build", + "coverage", + ".tox", + ".mypy_cache", + ".ruff_cache", + ".pytest_cache", + } +) + + +class PythonAdapter: + """High-confidence Python AST adapter wrapping ``build_python_map``.""" + + language = "python" + extensions = frozenset({".py"}) + fidelity = "ast" + note = ( + "Python AST maps with McCabe complexity and high-confidence call edges " + "(ambiguous names omit rather than invent)." + ) + + def build_map(self, root: pathlib.Path) -> dict[str, Any]: + return build_python_map(root) + + +_ADAPTERS: list[LanguageAdapter] = [ + PythonAdapter(), +] + + +def registered_adapters() -> list[LanguageAdapter]: + return list(_ADAPTERS) + + +def adapter_for_extension(ext: str) -> LanguageAdapter | None: + needle = ext.lower() if ext.startswith(".") else f".{ext.lower()}" + for adapter in _ADAPTERS: + if needle in adapter.extensions: + return adapter + return None + + +def supported_extensions() -> frozenset[str]: + exts: set[str] = set() + for adapter in _ADAPTERS: + exts |= set(adapter.extensions) + return frozenset(exts) + + +def supported_languages() -> list[str]: + return [a.language for a in _ADAPTERS] + + +def _should_skip_dir(name: str) -> bool: + return name in _SKIP_DIR_NAMES or (name.startswith(".") and name not in {".", ".."}) + + +def _path_has_hidden_segment(path: pathlib.Path) -> bool: + """Match Python analyzer: skip paths with any ``.``-prefixed segment.""" + return any(part.startswith(".") for part in path.parts) + + +def discover_source_inventory( + root: pathlib.Path, +) -> tuple[dict[str, list[pathlib.Path]], dict[str, int]]: + """Walk ``root``; bucket supported files by language; count unsupported by ext. + + Unsupported files are **not** passed to any analyzer (no empty fake maps). + """ + by_lang: dict[str, list[pathlib.Path]] = {a.language: [] for a in _ADAPTERS} + unsupported: dict[str, int] = {} + + if root.is_file(): + paths = [root] + else: + paths = [] + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = [d for d in dirnames if not _should_skip_dir(d)] + for name in filenames: + paths.append(pathlib.Path(dirpath) / name) + + for path in paths: + if _path_has_hidden_segment(path): + continue + ext = path.suffix.lower() + adapter = adapter_for_extension(ext) + if adapter is not None: + by_lang.setdefault(adapter.language, []).append(path) + elif ext in _UNSUPPORTED_SOURCE_EXTS: + key = ext.lstrip(".") + unsupported[key] = unsupported.get(key, 0) + 1 + return by_lang, unsupported + + +def normalize_lang_filter(languages: Sequence[str] | None) -> set[str] | None: + """Map user ``--lang`` tokens to registry language ids (``typescript`` ΓåÆ ``javascript``).""" + if not languages: + return None + wanted: set[str] = set() + aliases = { + "js": "javascript", + "ts": "javascript", # TS handled by JS adapter + "typescript": "javascript", + "py": "python", + "golang": "go", + "c#": "csharp", + "cs": "csharp", + "csharp": "csharp", + } + for raw in languages: + for part in str(raw).split(","): + tok = part.strip().lower() + if not tok: + continue + wanted.add(aliases.get(tok, tok)) + return wanted + + +def build_repo_map( + root: pathlib.Path, + *, + languages: Sequence[str] | None = None, +) -> dict[str, Any]: + """Dispatch registered analyzers and merge into one repository map. + + Unsupported source files are counted under ``unsupported`` and never fed to + the Python (or other) analyzer as empty stand-ins. + """ + root = pathlib.Path(root) + by_lang, unsupported = discover_source_inventory(root) + lang_filter = normalize_lang_filter(languages) + + functions: dict[str, Any] = {} + analyzed: list[str] = [] + fidelity: dict[str, str] = {} + notes: dict[str, str] = {} + files_analyzed = 0 + # Last single-adapter partial metadata (propagated when exactly one language ran). + last_partial: dict[str, Any] | None = None + + for adapter in _ADAPTERS: + if lang_filter is not None and adapter.language not in lang_filter: + continue + files = by_lang.get(adapter.language) or [] + if not files and lang_filter is None: + # Still allow adapter walk (e.g. single-file roots); skip if inventory empty. + continue + if lang_filter is not None and not files: + continue + partial = adapter.build_map(root) + funcs = partial.get("functions") or {} + if not funcs and not files: + continue + functions.update(funcs) + analyzed.append(adapter.language) + # Prefer per-map fidelity when the adapter reports what actually ran + # (JS AST vs regex fallback). + partial_fid = partial.get("analyzer_fidelity") + if isinstance(partial_fid, str) and partial_fid: + fidelity[adapter.language] = partial_fid + else: + fidelity[adapter.language] = adapter.fidelity + notes[adapter.language] = partial.get("analyzer_note") or adapter.note + files_analyzed += len(files) + last_partial = partial + + # Language label + if len(analyzed) == 0: + language = "none" + elif len(analyzed) == 1: + # Prefer typescript label when JS adapter says so and only that adapter ran. + if analyzed[0] == "javascript": + language = "javascript" + # Re-check suffixes for typescript-only trees. + js_files = by_lang.get("javascript") or [] + suffixes = {p.suffix.lower() for p in js_files} + if suffixes and suffixes <= {".ts", ".tsx"}: + language = "typescript" + # Honor adapter language when it already distinguished typescript. + if last_partial and last_partial.get("language") in {"javascript", "typescript"}: + language = str(last_partial["language"]) + else: + language = analyzed[0] + else: + language = "mixed" + + if analyzed == ["python"]: + complexity_kind = "mccabe" + elif analyzed == ["javascript"] and last_partial is not None: + complexity_kind = str(last_partial.get("complexity_kind") or "keyword-heuristic") + elif analyzed == ["go"] and last_partial is not None: + complexity_kind = str(last_partial.get("complexity_kind") or "keyword-heuristic") + elif analyzed == ["java"] and last_partial is not None: + complexity_kind = str(last_partial.get("complexity_kind") or "keyword-heuristic") + elif analyzed == ["rust"] and last_partial is not None: + complexity_kind = str(last_partial.get("complexity_kind") or "keyword-heuristic") + elif analyzed == ["csharp"] and last_partial is not None: + complexity_kind = str(last_partial.get("complexity_kind") or "keyword-heuristic") + elif analyzed: + complexity_kind = "mixed" + else: + complexity_kind = "none" + + out: dict[str, Any] = { + "language": language, + "languages_analyzed": analyzed, + "analyzer_fidelity": fidelity, + "analyzer_notes": notes, + "unsupported": unsupported, + "files_analyzed": files_analyzed, + "functions": functions, + "complexity_kind": complexity_kind, + } + # Single-adapter maps: surface honest analyzer id (e.g. javascript-best-effort). + if len(analyzed) == 1 and last_partial is not None: + if "analyzer" in last_partial: + out["analyzer"] = last_partial["analyzer"] + elif analyzed[0] == "python": + out["analyzer"] = "python-ast" + if "analyzer_note" in last_partial: + out["analyzer_note"] = last_partial["analyzer_note"] + return out + + +def format_unsupported_summary(unsupported: dict[str, int]) -> str: + if not unsupported: + return "" + parts = [f".{ext}├ù{count}" for ext, count in sorted(unsupported.items())] + return ( + "Not analyzed (no adapter): " + + ", ".join(parts) + + ". These files were skipped ΓÇö not scanned as Python or empty maps." + ) + + +def iter_adapter_notes() -> Iterable[tuple[str, str, str]]: + for a in _ADAPTERS: + yield a.language, a.fidelity, a.note diff --git a/cli/ucli/boundaries/scan.py b/cli/ucli/boundaries/scan.py index d23569b..6681f8e 100644 --- a/cli/ucli/boundaries/scan.py +++ b/cli/ucli/boundaries/scan.py @@ -1,27 +1,33 @@ -import os, re -from typing import Dict, Any +import os +import re +from typing import Any -def scan_boundaries(path: str) -> Dict[str, Any]: + +def scan_boundaries(path: str) -> dict[str, Any]: endpoints = [] protos = [] tables = [] - for root, dirs, files in os.walk(path): + for root, _dirs, files in os.walk(path): for f in files: full = os.path.join(root, f) try: - head = open(full, 'r', encoding='utf-8', errors='ignore').read(2000) - except Exception: + head = open(full, encoding="utf-8", errors="ignore").read(2000) + except OSError: continue - if f.endswith(('.yml', '.yaml')) and 'openapi' in head.lower(): - data = open(full, 'r', encoding='utf-8', errors='ignore').read() + if f.endswith((".yml", ".yaml")) and "openapi" in head.lower(): + data = open(full, encoding="utf-8", errors="ignore").read() for m in re.finditer(r"\n\s{0,6}(/[^\s:]+):", data): endpoints.append(m.group(1)) - elif f.endswith('.proto'): - txt = open(full, 'r', encoding='utf-8', errors='ignore').read() + elif f.endswith(".proto"): + txt = open(full, encoding="utf-8", errors="ignore").read() for m in re.finditer(r"rpc\s+(\w+)\s*\(", txt): protos.append(m.group(1)) - elif f.endswith('.sql'): - txt = open(full, 'r', encoding='utf-8', errors='ignore').read() + elif f.endswith(".sql"): + txt = open(full, encoding="utf-8", errors="ignore").read() for m in re.finditer(r"CREATE\s+TABLE\s+(\w+)", txt, re.I): tables.append(m.group(1)) - return {'openapi_paths': sorted(set(endpoints)), 'proto_rpcs': sorted(set(protos)), 'sql_tables': sorted(set(tables))} + return { + "openapi_paths": sorted(set(endpoints)), + "proto_rpcs": sorted(set(protos)), + "sql_tables": sorted(set(tables)), + } diff --git a/cli/ucli/commands/__init__.py b/cli/ucli/commands/__init__.py new file mode 100644 index 0000000..58c540b --- /dev/null +++ b/cli/ucli/commands/__init__.py @@ -0,0 +1,6 @@ +"""Extracted CLI command helpers. + +Business logic lives in ``*_ops`` modules. Typer group wiring for lens/trace/ +boundaries/contracts/pack/visual lives in ``typer_groups``; top-level commands +and the app entrypoint stay in ``main.py``. +""" diff --git a/cli/ucli/commands/ci_ops.py b/cli/ucli/commands/ci_ops.py new file mode 100644 index 0000000..b5ad41f --- /dev/null +++ b/cli/ucli/commands/ci_ops.py @@ -0,0 +1,241 @@ +"""CI command: scan + lens + tour artifacts for pipelines (extracted from main).""" + +from __future__ import annotations + +import json +import os +import pathlib +from typing import Any + +import typer +from rich.console import Console +from rich.progress import ( + BarColumn, + Progress, + SpinnerColumn, + TextColumn, + TimeElapsedColumn, +) + +from ucli.analyzers.registry import build_repo_map +from ucli.lens.lens import lens_from_seeds, rank_by_error_proximity, write_tour_md +from ucli.metrics.ttu import record as ttu_record + + +def _complexity_value(raw: Any) -> int: + if isinstance(raw, dict): + return int(raw.get("cyclomatic", 0) or 0) + try: + return int(raw or 0) + except (TypeError, ValueError): + return 0 + + +def generate_ci_report(repo_map: dict[str, Any], lens: dict[str, Any], scan_path: str) -> str: + """Generate comprehensive CI report markdown.""" + functions = repo_map.get("functions", {}) + functions_count = len(functions) + + total_complexity = sum( + _complexity_value(func.get("complexity", 0)) for func in functions.values() + ) + avg_complexity = total_complexity / functions_count if functions_count > 0 else 0 + + high_complexity_count = sum( + 1 for func in functions.values() if _complexity_value(func.get("complexity", 0)) > 10 + ) + side_effects_count = sum(1 for func in functions.values() if func.get("side_effects", [])) + + report = f"""# Understand-First CI Report + +## Analysis Summary + +- **Scan Path**: `{scan_path}` +- **Functions Analyzed**: {functions_count} +- **Total Complexity**: {total_complexity} +- **Average Complexity**: {avg_complexity:.1f} +- **High Complexity Functions**: {high_complexity_count} +- **Functions with Side Effects**: {side_effects_count} + +## Understanding Lens + +- **Seeds**: {len(lens.get("lens", {}).get("seeds", []))} +- **Functions in Lens**: {len(lens.get("functions", {}))} + +## Recommendations + +""" + + if high_complexity_count > 0: + report += f"- Consider refactoring {high_complexity_count} high-complexity functions\n" + + if side_effects_count > 0: + report += f"- Review {side_effects_count} functions with side effects\n" + + if avg_complexity > 5: + report += "- Overall complexity is high - consider architectural improvements\n" + + if functions_count == 0: + report += "- No functions found - check scan path and file patterns\n" + + report += """ +## Next Steps + +1. Review the understanding tour for key insights +2. Address any high-complexity functions +3. Document side effects and their implications +4. Update understanding artifacts as code evolves + +*Generated by Understand-First CI* +""" + + return report + + +def run_ci( + scan_path: str, + output_dir: str, + *, + fail_on_issues: bool = True, + generate_report: bool = True, +) -> None: + """Run Understand-First analysis for CI/CD pipeline with enhanced reporting.""" + console = Console() + + try: + os.makedirs(output_dir, exist_ok=True) + + console.print("[bold blue]🔍 Scanning repository...[/bold blue]") + repo_map_path = os.path.join(output_dir, "repo.json") + + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + BarColumn(), + TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), + TimeElapsedColumn(), + console=console, + ) as progress: + scan_task = progress.add_task("Scanning codebase...", total=100) + progress.update(scan_task, advance=50) + + result = build_repo_map(pathlib.Path(scan_path)) + progress.update(scan_task, advance=50) + + with open(repo_map_path, "w", encoding="utf-8") as f: + json.dump(result, f, indent=2) + + console.print(f"[green]✓ Repository map generated: {repo_map_path}[/green]") + + console.print("[bold blue]🎯 Generating understanding lens...[/bold blue]") + lens_path = os.path.join(output_dir, "lens.json") + + functions = list(result.get("functions", {}).keys()) + seeds = functions[:3] if functions else [] + + if seeds: + lens = lens_from_seeds(seeds, result) + rank_by_error_proximity(lens) + + with open(lens_path, "w", encoding="utf-8") as f: + json.dump(lens, f, indent=2) + + console.print(f"[green]✓ Understanding lens generated: {lens_path}[/green]") + else: + console.print("[yellow]⚠ No functions found for lens generation[/yellow]") + lens = {"functions": {}, "lens": {"seeds": []}} + + console.print("[bold blue]🗺️ Generating understanding tour...[/bold blue]") + tour_path = os.path.join(output_dir, "tour.md") + + if lens.get("functions"): + tour_md = write_tour_md(lens) + with open(tour_path, "w", encoding="utf-8") as f: + f.write(tour_md) + console.print(f"[green]✓ Understanding tour generated: {tour_path}[/green]") + else: + console.print("[yellow]⚠ No lens data for tour generation[/yellow]") + + if generate_report: + console.print("[bold blue]📊 Generating CI report...[/bold blue]") + report_path = os.path.join(output_dir, "ci-report.md") + + report_content = generate_ci_report(result, lens, scan_path) + with open(report_path, "w", encoding="utf-8") as f: + f.write(report_content) + console.print(f"[green]✓ CI report generated: {report_path}[/green]") + + issues: list[str] = [] + functions_count = len(result.get("functions", {})) + + if functions_count == 0: + issues.append("No functions found in codebase") + + high_complexity = [] + for func_name, func_data in result.get("functions", {}).items(): + complexity = _complexity_value(func_data.get("complexity", 0)) + if complexity > 10: + high_complexity.append(f"{func_name} (complexity: {complexity})") + + if high_complexity: + issues.append(f"High complexity functions found: {', '.join(high_complexity[:3])}") + + side_effects = [] + for func_name, func_data in result.get("functions", {}).items(): + effects = func_data.get("side_effects", []) + if effects: + side_effects.append(f"{func_name}: {', '.join(effects)}") + + if side_effects: + issues.append(f"Functions with side effects: {', '.join(side_effects[:3])}") + + if issues: + console.print("\n[yellow]⚠ Issues detected:[/yellow]") + for issue in issues: + console.print(f" • {issue}") + + if fail_on_issues: + console.print("\n[red]❌ CI failed due to understanding issues[/red]") + raise typer.Exit(1) from None + console.print("\n[yellow]⚠ CI completed with warnings[/yellow]") + else: + console.print("\n[green]✅ CI completed successfully - no issues detected[/green]") + + summary = { + "scan_path": scan_path, + "output_dir": output_dir, + "functions_analyzed": functions_count, + "issues_found": len(issues), + "artifacts_generated": [ + a + for a in [ + "repo.json", + "lens.json" if seeds else None, + "tour.md" if lens.get("functions") else None, + "ci-report.md" if generate_report else None, + ] + if a is not None + ], + "issues": issues, + } + + summary_path = os.path.join(output_dir, "ci-summary.json") + with open(summary_path, "w", encoding="utf-8") as f: + json.dump(summary, f, indent=2) + + console.print(f"\n[green]✓ CI analysis completed. Summary: {summary_path}[/green]") + + ttu_record( + "ci_completed", + { + "functions_analyzed": functions_count, + "issues_found": len(issues), + "artifacts_generated": len(summary["artifacts_generated"]), + }, + ) + + except typer.Exit: + raise + except Exception as e: + console.print(f"[red]Error during CI analysis:[/red] {str(e)}") + raise typer.Exit(1) from None diff --git a/cli/ucli/commands/contracts_ops.py b/cli/ucli/commands/contracts_ops.py new file mode 100644 index 0000000..6cfc3b9 --- /dev/null +++ b/cli/ucli/commands/contracts_ops.py @@ -0,0 +1,120 @@ +"""Contracts command implementations (extracted from main.py).""" + +from __future__ import annotations + +import json +import os +import pathlib + +import typer +from rich import print + +from ucli.contracts.contracts import ( + check_contracts, + compose, + from_openapi, + from_proto, + init_contracts, + lean_stubs, + report_json, + stub_tests, + verify_lean, +) + + +def run_contracts_init(path: str, o: str) -> None: + os.makedirs(pathlib.Path(o).parent, exist_ok=True) + txt = init_contracts(path) + with open(o, "w", encoding="utf-8") as f: + f.write(txt) + print(f"[green]Wrote[/green] {o}") + + +def run_contracts_check(path: str) -> None: + ok, report = check_contracts(path) + print(report) + if not ok: + raise typer.Exit(1) from None + + +def run_contracts_stub(path: str, o: str) -> None: + os.makedirs(pathlib.Path(o).parent, exist_ok=True) + txt = stub_tests(path) + with open(o, "w", encoding="utf-8") as f: + f.write(txt) + print(f"[green]Wrote[/green] {o}") + + +def run_contracts_from_openapi(path: str, o: str) -> None: + txt = from_openapi(path) + os.makedirs(pathlib.Path(o).parent, exist_ok=True) + with open(o, "w", encoding="utf-8") as f: + f.write(txt) + print(f"[green]Wrote[/green] {o}") + + +def run_contracts_from_proto(path: str, o: str) -> None: + txt = from_proto(path) + os.makedirs(pathlib.Path(o).parent, exist_ok=True) + with open(o, "w", encoding="utf-8") as f: + f.write(txt) + print(f"[green]Wrote[/green] {o}") + + +def run_contracts_lean_stubs(contracts_yaml: str, o: str) -> None: + os.makedirs(o, exist_ok=True) + count = lean_stubs(contracts_yaml, o) + print( + f"[green]Wrote[/green] {count} Lean scaffold stub(s) to {o} " + "(Prop := True / by trivial; not compiled)" + ) + + +def run_contracts_compose(inputs: list[str], o: str) -> None: + if not inputs: + print("[red]No input files provided[/red]") + raise typer.Exit(1) from None + txt = compose(inputs) + os.makedirs(pathlib.Path(o).parent, exist_ok=True) + with open(o, "w", encoding="utf-8") as f: + f.write(txt) + print(f"[green]Wrote[/green] {o}") + + +def run_contracts_verify_lean( + contracts_yaml: str, + lean_dir: str, + *, + json_out: bool = False, +) -> None: + data = verify_lean(contracts_yaml, lean_dir) + if json_out: + print(json.dumps(data, indent=2)) + else: + print( + f"mode: {data.get('mode', 'presence-only')} " + "(file presence only; does not compile Lean)\n" + f"modules: {data.get('modules_total')}\n" + f"functions: {data.get('functions_total')}" + ) + missing = data.get("missing_invariants", []) + if missing: + print("missing invariants:") + for m in missing: + print(f" - {m}") + if data.get("missing_invariants"): + raise typer.Exit(1) from None + + +def run_contracts_report(path: str, *, json_out: bool = False) -> None: + data = report_json(path) + if json_out: + print(json.dumps(data, indent=2)) + else: + for i in data.get("issues", []): + print( + f"[{i.get('severity', 'info')}] " + f"{i.get('module_path')}:{i.get('function')} - {i.get('message')}" + ) + if data.get("issues"): + raise typer.Exit(1) from None diff --git a/cli/ucli/commands/demo_ops.py b/cli/ucli/commands/demo_ops.py new file mode 100644 index 0000000..4284a8d --- /dev/null +++ b/cli/ucli/commands/demo_ops.py @@ -0,0 +1,70 @@ +"""Demo command: end-to-end sample artifacts (extracted from main.py).""" + +from __future__ import annotations + +import json +import os +import pathlib +import subprocess # nosec B404 # intentional: demo server fixed argv, no shell +import sys + +from rich import print + +from ucli.analyzers.python_analyzer import build_python_map +from ucli.contracts.contracts import from_openapi +from ucli.dashboard.build import build_dashboard +from ucli.lens.lens import ( + lens_from_seeds, + merge_trace_into_lens, + rank_by_error_proximity, + write_tour_md, +) +from ucli.trace.pytrace import run_callable_with_trace + + +def run_demo() -> None: + """Generate sample contracts, maps, lens, tour, and dashboard artifacts.""" + try: + txt = from_openapi("examples/apis/petstore-mini.yaml") + os.makedirs("contracts", exist_ok=True) + pathlib.Path("contracts/contracts_from_openapi.yaml").write_text(txt, encoding="utf-8") + except (OSError, ValueError, TypeError, KeyError): + # Optional OpenAPI sample may be absent in slim checkouts. + pass + + http_proc = subprocess.Popen( # nosec B603 + [sys.executable, "examples/servers/http_server.py"] + ) + try: + os.makedirs("traces", exist_ok=True) + data = run_callable_with_trace("examples/app/hot_path.py", "run_hot_path") + pathlib.Path("traces/tour.json").write_text(json.dumps(data, indent=2), encoding="utf-8") + + os.makedirs("maps", exist_ok=True) + repo_map = build_python_map(pathlib.Path("examples/python_toy")) + pathlib.Path("maps/repo.json").write_text(json.dumps(repo_map, indent=2), encoding="utf-8") + lens = lens_from_seeds(["compute"], repo_map) + merged = merge_trace_into_lens(lens, data) + rank_by_error_proximity(merged) + pathlib.Path("maps/lens_merged.json").write_text( + json.dumps(merged, indent=2), encoding="utf-8" + ) + + os.makedirs("tours", exist_ok=True) + pathlib.Path("tours/demo.md").write_text(write_tour_md(merged), encoding="utf-8") + os.makedirs("docs", exist_ok=True) + pathlib.Path("docs/understanding-dashboard.md").write_text( + build_dashboard( + { + "repo": "maps/repo.json", + "lens": "maps/lens_merged.json", + "bounds": "maps/boundaries.json", + } + ), + encoding="utf-8", + ) + + url = f"file://{pathlib.Path('tours/demo.md').resolve()}" + print(f"[green]Open tour:[/green] {url}") + finally: + http_proc.terminate() diff --git a/cli/ucli/commands/diff_ops.py b/cli/ucli/commands/diff_ops.py new file mode 100644 index 0000000..1d990a9 --- /dev/null +++ b/cli/ucli/commands/diff_ops.py @@ -0,0 +1,496 @@ +"""Map/lens delta computation and `u diff` presentation (extracted from main). + +Complexity is McCabe cyclomatic from Python AST maps. JS/TS maps contribute +keyword-heuristic complexity only — deltas label fidelity so totals are not +misread as uniform McCabe. Side-effect diffs use heuristic `side_effects` +lists when present (Python analyzer); JS maps typically leave them empty. +""" + +from __future__ import annotations + +import json +import os +import pathlib +import traceback +from datetime import datetime +from typing import Any + +import typer +from rich.console import Console +from rich.panel import Panel +from rich.progress import ( + BarColumn, + Progress, + SpinnerColumn, + TextColumn, + TimeElapsedColumn, +) +from rich.table import Table + +from ucli.analyzers.map_meta import complexity_semantics_note, map_fidelity_label +from ucli.metrics.ttu import record as ttu_record +from ucli.visual.delta import lens_delta_svg + + +def _maps_complexity_note(old_data: dict[str, Any], new_data: dict[str, Any]) -> str: + notes = { + complexity_semantics_note(old_data), + complexity_semantics_note(new_data), + } + labels = {map_fidelity_label(old_data), map_fidelity_label(new_data)} + kinds = {old_data.get("complexity_kind"), new_data.get("complexity_kind")} + if labels <= {"ast"} and kinds <= {"mccabe", None}: + return "Complexity = McCabe cyclomatic on Python AST maps." + if labels <= {"ast"} and "ast-cyclomatic" in kinds: + return ( + "Complexity = AST decision-point cyclomatic (JS/TS ast-cyclomatic and/or " + "Python McCabe); check analyzer ids — not interchangeable totals." + ) + if labels <= {"best-effort"}: + return "Complexity = keyword-heuristic on JS/TS best-effort maps (not McCabe)." + if "mixed" in labels or labels == {"ast", "best-effort"}: + return ( + "Complexity mixes AST cyclomatic and keyword-heuristic paths; " + "net totals are not uniform McCabe." + ) + return " ".join(sorted(notes)) + + +def compute_map_delta( + old_data: dict[str, Any], + new_data: dict[str, Any], + *, + policy_threshold: int = 5, + old_label: str = "", + new_label: str = "", +) -> dict[str, Any]: + """Compare two map/lens JSON documents and return a structured delta.""" + old_functions = set(old_data.get("functions", {}).keys()) + new_functions = set(new_data.get("functions", {}).keys()) + added_functions = sorted(new_functions - old_functions) + removed_functions = sorted(old_functions - new_functions) + modified_functions: list[str] = [] + policy_breaches: list[dict[str, Any]] = [] + complexity_increased: list[dict[str, Any]] = [] + complexity_decreased: list[dict[str, Any]] = [] + side_effects_added: list[dict[str, Any]] = [] + side_effects_removed: list[dict[str, Any]] = [] + + def _cc(meta: dict[str, Any]) -> int: + c = meta.get("complexity", 0) + if isinstance(c, dict): + return int(c.get("cyclomatic", 0) or 0) + return int(c or 0) + + def _se(meta: dict[str, Any]) -> list[str]: + raw = meta.get("side_effects") + if raw is None: + return [] + if isinstance(raw, list): + return [str(x) for x in raw] + if isinstance(raw, (int, float)): + return [] + return [] + + total_old = sum(_cc(old_data["functions"][f]) for f in old_functions) + total_new = sum(_cc(new_data["functions"][f]) for f in new_functions) + for func in sorted(old_functions & new_functions): + old_func = old_data["functions"][func] + new_func = new_data["functions"][func] + old_c, new_c = _cc(old_func), _cc(new_func) + old_se, new_se = _se(old_func), _se(new_func) + changed = ( + old_c != new_c + or old_se != new_se + or old_func.get("lines", 0) != new_func.get("lines", 0) + ) + if not changed: + continue + modified_functions.append(func) + if new_c > old_c: + complexity_increased.append( + {"function": func, "old_complexity": old_c, "new_complexity": new_c} + ) + elif new_c < old_c: + complexity_decreased.append( + {"function": func, "old_complexity": old_c, "new_complexity": new_c} + ) + se_add = sorted(set(new_se) - set(old_se)) + se_rem = sorted(set(old_se) - set(new_se)) + if se_add: + side_effects_added.append({"function": func, "tags": se_add}) + if se_rem: + side_effects_removed.append({"function": func, "tags": se_rem}) + if new_c > policy_threshold: + policy_breaches.append( + { + "function": func, + "old_complexity": old_c, + "new_complexity": new_c, + "threshold": policy_threshold, + } + ) + for func in added_functions: + func_data = new_data["functions"][func] + new_c = _cc(func_data) + if new_c > 0: + complexity_increased.append( + {"function": func, "old_complexity": 0, "new_complexity": new_c} + ) + se = _se(func_data) + if se: + side_effects_added.append({"function": func, "tags": se}) + if new_c > policy_threshold: + policy_breaches.append( + { + "function": func, + "old_complexity": 0, + "new_complexity": new_c, + "threshold": policy_threshold, + } + ) + for func in removed_functions: + old_c = _cc(old_data["functions"][func]) + if old_c > 0: + complexity_decreased.append( + {"function": func, "old_complexity": old_c, "new_complexity": 0} + ) + se = _se(old_data["functions"][func]) + if se: + side_effects_removed.append({"function": func, "tags": se}) + net_change = total_new - total_old + return { + "summary": { + "added": len(added_functions), + "removed": len(removed_functions), + "modified": len(modified_functions), + "total_changes": len(added_functions) + + len(removed_functions) + + len(modified_functions), + "policy_breaches": len(policy_breaches), + "policy_threshold": policy_threshold, + "complexity_net_change": net_change, + "complexity_total_old": total_old, + "complexity_total_new": total_new, + "side_effects_added": len(side_effects_added), + "side_effects_removed": len(side_effects_removed), + "complexity_note": _maps_complexity_note(old_data, new_data), + "old_fidelity": map_fidelity_label(old_data), + "new_fidelity": map_fidelity_label(new_data), + }, + "added_functions": added_functions, + "removed_functions": removed_functions, + "modified_functions": modified_functions, + "policy_breaches": policy_breaches, + "complexity_delta": { + "net_change": net_change, + "total_old": total_old, + "total_new": total_new, + "increased": complexity_increased, + "decreased": complexity_decreased, + "note": _maps_complexity_note(old_data, new_data), + }, + "side_effects_added": side_effects_added, + "side_effects_removed": side_effects_removed, + "old_lens": old_label, + "new_lens": new_label, + "timestamp": datetime.now().isoformat(), + } + + +def generate_enhanced_diff_markdown( + added_functions, + removed_functions, + modified_functions, + policy_breaches, + changes_count, + policy_threshold, +) -> str: + """Generate enhanced Markdown summary of diff analysis with policy breach information.""" + content = "# Delta Analysis Summary\n\n" + content += f"**Total Changes:** {changes_count}\n" + content += f"**Policy Breaches:** {len(policy_breaches)}\n" + content += f"**Policy Threshold:** {policy_threshold}\n\n" + if policy_breaches: + content += "## Policy Breaches Detected\n\n" + content += ( + f"The following functions exceed the complexity threshold of {policy_threshold}:\n\n" + ) + for breach in policy_breaches: + content += ( + f"- **{breach['function']}**: {breach['old_complexity']} → " + f"{breach['new_complexity']} (threshold: {breach['threshold']})\n" + ) + content += "\n" + elif changes_count > 0: + content += "## Changes Detected\n\n" + content += "The following changes were detected in your codebase:\n\n" + else: + content += "## No Changes Detected\n\n" + content += "Your codebase has no changes since the last analysis.\n\n" + if added_functions: + content += f"## Added Functions ({len(added_functions)})\n\n" + for func in sorted(added_functions): + content += f"- `{func}`\n" + content += "\n" + if removed_functions: + content += f"## Removed Functions ({len(removed_functions)})\n\n" + for func in sorted(removed_functions): + content += f"- `{func}`\n" + content += "\n" + if modified_functions: + content += f"## Modified Functions ({len(modified_functions)})\n\n" + for func in sorted(modified_functions): + content += f"- `{func}`\n" + content += "\n" + content += "## Review Checklist\n\n" + content += "- [ ] Review added functions for new side effects\n" + content += "- [ ] Verify removed functions don't break dependencies\n" + content += "- [ ] Check modified functions for complexity changes\n" + if policy_breaches: + content += f"- [ ] **Address policy breaches (complexity > {policy_threshold})**\n" + content += "- [ ] Update documentation if needed\n" + content += "- [ ] Update understanding tours if needed\n" + content += "- [ ] Update CI/CD policies if needed\n\n" + content += "## Exit Codes\n\n" + content += "The following exit codes are used for CI integration:\n\n" + content += "- **0**: No changes detected\n" + content += "- **2**: Changes detected (update tours/documentation)\n" + content += "- **3**: Policy breaches detected (address complexity issues)\n\n" + content += "*Generated by Understand-First*" + return content + + +def run_diff( + old_lens: str, + new_lens: str, + o: str, + *, + verbose: bool = False, + json_output: bool = False, + markdown: bool = False, + ci_gate: bool = False, + policy_threshold: int = 5, +) -> None: + """Compare two lens files and generate delta visualization with enhanced analysis.""" + console = Console() + try: + if not pathlib.Path(old_lens).exists(): + console.print( + Panel.fit( + f"[red]Error: Old lens file '{old_lens}' does not exist[/red]\n\n" + "This could mean:\n" + "• The file path is incorrect\n" + "• The file hasn't been generated yet\n" + "• There's a typo in the filename", + title="File Not Found", + border_style="red", + ) + ) + raise typer.Exit(1) from None + if not pathlib.Path(new_lens).exists(): + console.print( + Panel.fit( + f"[red]Error: New lens file '{new_lens}' does not exist[/red]\n\n" + "This could mean:\n" + "• The file path is incorrect\n" + "• The file hasn't been generated yet\n" + "• There's a typo in the filename", + title="File Not Found", + border_style="red", + ) + ) + raise typer.Exit(1) from None + try: + with open(old_lens, encoding="utf-8") as f: + old_data = json.load(f) + except json.JSONDecodeError as e: + console.print(f"[red]Error parsing old lens file:[/red] {str(e)}") + raise typer.Exit(1) from None + try: + with open(new_lens, encoding="utf-8") as f: + new_data = json.load(f) + except json.JSONDecodeError as e: + console.print(f"[red]Error parsing new lens file:[/red] {str(e)}") + raise typer.Exit(1) from None + config_table = Table(title="Delta Analysis Configuration", show_header=False) + config_table.add_column("Setting", style="cyan") + config_table.add_column("Value", style="white") + config_table.add_row("Old Lens", old_lens) + config_table.add_row("New Lens", new_lens) + config_table.add_row("Output Format", "JSON" if json_output else "SVG") + config_table.add_row("Output File", o) + config_table.add_row("Policy Threshold", str(policy_threshold)) + config_table.add_row("CI Gate", "Enabled" if ci_gate else "Disabled") + console.print(config_table) + console.print() + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + BarColumn(), + TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), + TimeElapsedColumn(), + console=console, + ) as progress: + diff_task = progress.add_task("Analyzing differences...", total=100) + progress.update(diff_task, advance=10) + progress.update(diff_task, advance=20) + delta_data = compute_map_delta( + old_data, + new_data, + policy_threshold=policy_threshold, + old_label=old_lens, + new_label=new_lens, + ) + added_functions = set(delta_data["added_functions"]) + removed_functions = set(delta_data["removed_functions"]) + modified_functions = list(delta_data["modified_functions"]) + policy_breaches = list(delta_data["policy_breaches"]) + progress.update(diff_task, advance=50) + if json_output: + os.makedirs(pathlib.Path(o).parent, exist_ok=True) + with open(o, "w", encoding="utf-8") as f: + json.dump(delta_data, f, indent=2) + else: + svg = lens_delta_svg(old_lens, new_lens) + with open(o, "w", encoding="utf-8") as df: + df.write(svg) + progress.update(diff_task, advance=20) + changes_count = len(added_functions) + len(removed_functions) + len(modified_functions) + if policy_breaches: + border_style = "red" + status_icon = "!" + status_text = "Policy breaches detected!" + elif changes_count > 0: + border_style = "yellow" + status_icon = "~" + status_text = "Changes detected" + else: + border_style = "green" + status_icon = "OK" + status_text = "No changes detected" + net_cc = delta_data.get("summary", {}).get("complexity_net_change", 0) + results_panel = Panel( + f"[green]Delta analysis completed![/green]\n\n" + f"[bold]Changes Summary:[/bold]\n" + f" • Functions added: {len(added_functions)}\n" + f" • Functions removed: {len(removed_functions)}\n" + f" • Functions modified: {len(modified_functions)}\n" + f" • Total changes: {changes_count}\n" + f" • Complexity net change: {net_cc:+d}\n" + f" • Policy breaches: {len(policy_breaches)}\n" + f" • Output file: {o}\n\n" + f"[bold]Review Impact:[/bold]\n" + f" • Check added functions for new side effects\n" + f" • Verify removed functions don't break dependencies\n" + f" • Review modified functions for complexity changes\n" + f" • Address policy breaches (complexity > {policy_threshold})", + title=f"{status_icon} Delta Analysis Results - {status_text}", + border_style=border_style, + ) + console.print(results_panel) + if policy_breaches: + console.print("\n[bold red]Policy Breaches Detected:[/bold red]") + breach_table = Table(title="Functions Exceeding Complexity Threshold") + breach_table.add_column("Function", style="cyan") + breach_table.add_column("Old Complexity", justify="center", style="yellow") + breach_table.add_column("New Complexity", justify="center", style="red") + breach_table.add_column("Threshold", justify="center", style="magenta") + for breach in policy_breaches: + breach_table.add_row( + breach["function"], + str(breach["old_complexity"]), + str(breach["new_complexity"]), + str(breach["threshold"]), + ) + console.print(breach_table) + if verbose and changes_count > 0: + if added_functions: + console.print("\n[bold green]Added Functions:[/bold green]") + for func in sorted(added_functions): + func_data = new_data["functions"][func] + complexity = func_data.get("complexity", 0) + side_effects = func_data.get("side_effects", []) + effects_str = ( + f" (side effects: {', '.join(side_effects)})" if side_effects else "" + ) + console.print(f" + {func} (complexity: {complexity}){effects_str}") + if removed_functions: + console.print("\n[bold red]Removed Functions:[/bold red]") + for func in sorted(removed_functions): + console.print(f" - {func}") + if modified_functions: + console.print("\n[bold yellow]Modified Functions:[/bold yellow]") + for func in sorted(modified_functions): + old_func = old_data["functions"][func] + new_func = new_data["functions"][func] + old_complexity = old_func.get("complexity", 0) + new_complexity = new_func.get("complexity", 0) + console.print(f" ~ {func} (complexity: {old_complexity} → {new_complexity})") + ttu_record( + "diff_analyzed", + { + "changes_count": changes_count, + "added": len(added_functions), + "removed": len(removed_functions), + "modified": len(modified_functions), + "policy_breaches": len(policy_breaches), + }, + ) + if markdown: + markdown_content = generate_enhanced_diff_markdown( + added_functions, + removed_functions, + modified_functions, + policy_breaches, + changes_count, + policy_threshold, + ) + markdown_path = o.replace(".svg", ".md").replace(".json", ".md") + with open(markdown_path, "w", encoding="utf-8") as f: + f.write(markdown_content) + console.print(f"[green]Markdown summary written to {markdown_path}[/green]") + if ci_gate: + if policy_breaches: + console.print( + f"\n[red]Policy breaches detected! {len(policy_breaches)} functions " + f"exceed complexity threshold of {policy_threshold}.[/red]" + ) + raise typer.Exit(3) + if changes_count > 0: + console.print( + "\n[yellow]Changes detected. Consider updating tours and documentation.[/yellow]" + ) + raise typer.Exit(2) + console.print("\n[green]No changes detected.[/green]") + raise typer.Exit(0) + if policy_breaches: + console.print( + f"\n[red]Policy breaches detected! {len(policy_breaches)} functions " + f"exceed complexity threshold of {policy_threshold}.[/red]" + ) + elif changes_count > 0: + console.print( + "\n[yellow]Changes detected. Consider updating tours and documentation.[/yellow]" + ) + else: + console.print("\n[green]No changes detected.[/green]") + raise typer.Exit(0) from None + except typer.Exit: + raise + except Exception as e: + console.print( + Panel.fit( + f"[red]Error during diff analysis:[/red] {str(e)}\n\n" + "This might be due to:\n" + "• Corrupted lens files\n" + "• File permission issues\n" + "• Invalid JSON format\n\n" + "Try running with --verbose for more details", + title="Analysis Error", + border_style="red", + ) + ) + if verbose: + console.print(f"[dim]{traceback.format_exc()}[/dim]") + raise typer.Exit(1) from None diff --git a/cli/ucli/commands/doctor_ops.py b/cli/ucli/commands/doctor_ops.py new file mode 100644 index 0000000..7a3b497 --- /dev/null +++ b/cli/ucli/commands/doctor_ops.py @@ -0,0 +1,218 @@ +"""Doctor command: environment sanity checks (extracted from main.py).""" + +from __future__ import annotations + +import pathlib +import shutil +import socket +import subprocess # nosec B404 # intentional: doctor probes node -v with fixed argv +import sys + +import typer +from rich import print + + +def run_doctor() -> None: + problems: list[str] = [] + notes: list[str] = [] + + def ok(msg: str) -> None: + print(f"[green]OK[/green] {msg}") + + def warn(msg: str, fix: str | None = None) -> None: + print(f"[yellow]WARN[/yellow] {msg}") + if fix: + notes.append(f"- {msg} → {fix}") + + def fail(msg: str, fix: str | None = None) -> None: + print(f"[red]FAIL[/red] {msg}") + problems.append(msg) + if fix: + notes.append(f"- {msg} → {fix}") + + ok(f"Python {sys.version.split()[0]}") + + node_bin = shutil.which("node") + if node_bin: + try: + r = subprocess.run( # nosec B603 + [node_bin, "-v"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + check=False, + ) + if r.returncode == 0: + ok(f"Node {r.stdout.strip()}") + try: + import importlib + + js_bridge = importlib.import_module("ucli.analyzers.js_ast_bridge") + if js_bridge.ast_backend_available(): + ok("JS/TS AST worker ready (typescript in cli/ucli/analyzers/js_ast)") + else: + warn( + "JS/TS AST worker deps missing — regex fallback will be used", + js_bridge.ast_unavailable_reason() + or "npm install in cli/ucli/analyzers/js_ast", + ) + except Exception: + warn( + "Could not probe JS/TS AST worker", + "npm install in cli/ucli/analyzers/js_ast", + ) + else: + warn( + "Node not found in PATH", + "Install Node 18+ or use Devcontainer/Codespaces", + ) + except OSError: + warn("Node not found in PATH", "Install Node 18+ or use Devcontainer/Codespaces") + else: + warn("Node not found in PATH", "Install Node 18+ or use Devcontainer/Codespaces") + + go_bin = shutil.which("go") + if go_bin: + try: + r = subprocess.run( # nosec B603 + [go_bin, "version"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + check=False, + ) + if r.returncode == 0: + ok(f"Go {r.stdout.strip()}") + try: + import importlib + + go_bridge = importlib.import_module("ucli.analyzers.go_ast_bridge") + if go_bridge.ast_backend_available(): + ok("Go AST worker ready (go run cli/ucli/analyzers/go_ast)") + else: + warn( + "Go AST worker not ready — regex fallback will be used", + go_bridge.ast_unavailable_reason() or "Install Go 1.21+", + ) + except Exception: + warn("Could not probe Go AST worker", "Install Go 1.21+") + else: + warn("Go not usable", "Install Go 1.21+ for go-ast maps") + except OSError: + warn("Go not found in PATH", "Install Go 1.21+ for go-ast maps") + else: + warn("Go not found in PATH", "Install Go 1.21+ for go-ast maps (regex fallback otherwise)") + + cargo_bin = shutil.which("cargo") + if cargo_bin: + try: + import importlib + + rust_bridge = importlib.import_module("ucli.analyzers.rust_ast_bridge") + if rust_bridge.ast_backend_available(): + ok("Rust AST worker ready (cargo + cli/ucli/analyzers/rust_ast)") + else: + warn( + "Rust AST worker not ready — regex fallback will be used", + rust_bridge.ast_unavailable_reason() or "Install Rust/cargo", + ) + except Exception: + warn("Could not probe Rust AST worker", "Install Rust/cargo") + else: + warn( + "cargo not found on PATH", + "Install Rust 1.70+ for rust-ast maps (regex fallback otherwise)", + ) + + try: + import importlib + + java_mod = importlib.import_module("ucli.analyzers.java_analyzer") + if java_mod.javalang_available(): + ok("javalang available (java-ast path)") + else: + warn( + "javalang not installed — Java scans use regex best-effort", + "pip install javalang or understand-first[analyzers]", + ) + except Exception: + warn("Could not probe javalang", "pip install javalang") + + dotnet_bin = shutil.which("dotnet") + if dotnet_bin: + try: + import importlib + + csharp_bridge = importlib.import_module("ucli.analyzers.csharp_ast_bridge") + if csharp_bridge.ast_backend_available(): + ok("C# AST worker ready (dotnet SDK + cli/ucli/analyzers/csharp_ast)") + else: + warn( + "C# AST worker not ready — regex fallback will be used", + csharp_bridge.ast_unavailable_reason() or "Install .NET SDK 8+", + ) + except Exception: + warn("Could not probe C# AST worker", "Install .NET SDK 8+") + else: + warn( + "dotnet not found on PATH", + "Install .NET SDK 8+ for csharp-ast maps (regex fallback otherwise)", + ) + + try: + import grpc_tools # noqa: F401 + except ImportError: + fail("grpc_tools missing", "pip install grpcio-tools") + else: + ok("grpc_tools available") + + def can_bind(port: int) -> bool: + try: + s = socket.socket() + s.bind(("127.0.0.1", port)) + s.close() + return True + except Exception: + return False + + if not can_bind(8000): + warn( + "Port 8000 not available", + "Stop the process using 8000 or change server port", + ) + else: + ok("Port 8000 available") + if not can_bind(50051): + warn( + "Port 50051 not available", + "Stop the process using 50051 or change gRPC port", + ) + else: + ok("Port 50051 available") + + code_bin = shutil.which("code") or shutil.which("code.cmd") + if code_bin: + ok(f"VS Code found at {code_bin}") + else: + warn("VS Code not found", "Install VS Code or use Codespaces") + + try: + test_file = pathlib.Path(".uf_write_test") + test_file.write_text("ok", encoding="utf-8") + test_file.unlink() + ok("Repo write permissions OK") + except Exception: + fail( + "No write permission in repo", + "Check filesystem permissions or workspace settings", + ) + + print("\nNext steps") + doc_url = "https://github.com/SentinelOps-CI/understand-first#readme" + for n in notes: + print(n) + if problems: + print(f"\nSee docs: {doc_url}") + raise typer.Exit(code=1) from None + print("All checks passed. See docs for advanced setup:") + print(doc_url) diff --git a/cli/ucli/commands/explorer_ops.py b/cli/ucli/commands/explorer_ops.py new file mode 100644 index 0000000..6a3b4c5 --- /dev/null +++ b/cli/ucli/commands/explorer_ops.py @@ -0,0 +1,531 @@ +"""Interactive TUI explorers (extracted from main.py).""" + +from __future__ import annotations + +import json +import pathlib +import time +from typing import Any + +import typer +from rich.console import Console +from rich.panel import Panel +from rich.progress import ( + BarColumn, + Progress, + SpinnerColumn, + TextColumn, + TimeElapsedColumn, +) +from rich.table import Table + +from ucli.analyzers.registry import build_repo_map +from ucli.lens.lens import lens_from_seeds, rank_by_error_proximity, write_tour_md +from ucli.metrics.ttu import record as ttu_record + + +def show_function_list(console: Console, functions: dict[str, Any]) -> None: + """Show paginated function list with enhanced formatting.""" + console.print("\n[bold cyan]Function List[/bold cyan]") + + page_size = 20 + total_pages = (len(functions) + page_size - 1) // page_size + page = 0 + + while True: + start_idx = page * page_size + end_idx = min(start_idx + page_size, len(functions)) + + table = Table(title=f"Functions (Page {page + 1} of {total_pages})") + table.add_column("Index", justify="center", style="cyan", width=5) + table.add_column("Function", style="cyan", no_wrap=True) + table.add_column("Complexity", justify="center", style="magenta", width=10) + table.add_column("Side Effects", justify="center", style="yellow", width=12) + table.add_column("Lines", justify="center", style="green", width=6) + + for i, (func_name, func_data) in enumerate(list(functions.items())[start_idx:end_idx]): + complexity = func_data.get("complexity", 0) + side_effects = func_data.get("side_effects", []) + lines = func_data.get("lines", 0) + + complexity_str = str(complexity) + if complexity > 5: + complexity_str = f"[red]{complexity}[/red]" + elif complexity > 3: + complexity_str = f"[yellow]{complexity}[/yellow]" + + effects_str = str(len(side_effects)) if side_effects else "0" + if side_effects: + effects_str = f"[yellow]{len(side_effects)}[/yellow]" + + table.add_row( + str(start_idx + i + 1), func_name, complexity_str, effects_str, str(lines) + ) + + console.print(table) + + console.print("\n[dim]Commands: n=next, p=previous, q=quit[/dim]") + cmd = typer.prompt("", default="q") + + if cmd.lower() == "q": + break + elif cmd.lower() == "n" and page < total_pages - 1: + page += 1 + elif cmd.lower() == "p" and page > 0: + page -= 1 + else: + console.print("[red]Invalid command. Use n, p, or q.[/red]") + + +def show_metrics(console: Console, result: dict[str, Any]) -> None: + """Show codebase metrics and statistics.""" + functions = result.get("functions", {}) + + if not functions: + console.print("[yellow]No functions to analyze metrics for.[/yellow]") + return + + total_functions = len(functions) + high_complexity = sum(1 for f in functions.values() if f.get("complexity", 0) > 5) + with_side_effects = sum(1 for f in functions.values() if f.get("side_effects", [])) + avg_complexity = sum(f.get("complexity", 0) for f in functions.values()) / total_functions + total_lines = sum(f.get("lines", 0) for f in functions.values()) + missing_docs = sum(1 for f in functions.values() if f.get("has_docstring") is False) + missing_types = sum(1 for f in functions.values() if f.get("has_type_hints") is False) + + metrics_table = Table(title="Codebase Metrics") + metrics_table.add_column("Metric", style="cyan") + metrics_table.add_column("Value", style="green") + metrics_table.add_column("Percentage", style="yellow") + + metrics_table.add_row("Total Functions", str(total_functions), "100%") + metrics_table.add_row( + "High Complexity", str(high_complexity), f"{high_complexity / total_functions * 100:.1f}%" + ) + metrics_table.add_row( + "With Side Effects", + str(with_side_effects), + f"{with_side_effects / total_functions * 100:.1f}%", + ) + metrics_table.add_row( + "Missing Docstrings", + str(missing_docs), + f"{missing_docs / total_functions * 100:.1f}%", + ) + metrics_table.add_row( + "Missing Type Hints", + str(missing_types), + f"{missing_types / total_functions * 100:.1f}%", + ) + metrics_table.add_row("Average Complexity", f"{avg_complexity:.1f}", "-") + metrics_table.add_row("Total Lines", str(total_lines), "-") + + console.print(metrics_table) + + complexity_dist: dict[str, int] = {} + for f in functions.values(): + complexity = f.get("complexity", 0) + if complexity <= 2: + complexity_dist["Low (1-2)"] = complexity_dist.get("Low (1-2)", 0) + 1 + elif complexity <= 5: + complexity_dist["Medium (3-5)"] = complexity_dist.get("Medium (3-5)", 0) + 1 + else: + complexity_dist["High (6+)"] = complexity_dist.get("High (6+)", 0) + 1 + + console.print("\n[bold cyan]Complexity Distribution[/bold cyan]") + for level, count in complexity_dist.items(): + percentage = count / total_functions * 100 + console.print(f" {level}: {count} functions ({percentage:.1f}%)") + + top_complex = sorted(functions.items(), key=lambda x: x[1].get("complexity", 0), reverse=True)[ + :5 + ] + if top_complex: + console.print("\n[bold cyan]Top 5 Most Complex Functions[/bold cyan]") + for i, (name, data) in enumerate(top_complex, 1): + complexity = data.get("complexity", 0) + console.print(f" {i}. {name} (complexity: {complexity})") + + +def search_functions(console: Console, functions: dict[str, Any]) -> None: + """Search functions by name or complexity with enhanced formatting.""" + console.print("\n[bold cyan]Function Search[/bold cyan]") + console.print("Search by function name or complexity threshold\n") + + query = typer.prompt("Enter search term (function name or complexity threshold)") + + if query.isdigit(): + threshold = int(query) + matches = [ + (name, data) + for name, data in functions.items() + if data.get("complexity", 0) >= threshold + ] + + if matches: + console.print( + Panel.fit( + f"[green]Found {len(matches)} functions with complexity >= {threshold}[/green]", + title="Search Results", + border_style="green", + ) + ) + + table = Table(title=f"Functions with complexity >= {threshold}") + table.add_column("Function", style="cyan", no_wrap=True) + table.add_column("Complexity", justify="center", style="magenta", width=10) + table.add_column("Side Effects", justify="center", style="yellow", width=12) + table.add_column("Lines", justify="center", style="green", width=6) + + for name, data in matches: + complexity = data.get("complexity", 0) + side_effects = data.get("side_effects", []) + lines = data.get("lines", 0) + + complexity_str = str(complexity) + if complexity > 5: + complexity_str = f"[red]{complexity}[/red]" + elif complexity > 3: + complexity_str = f"[yellow]{complexity}[/yellow]" + + effects_str = str(len(side_effects)) if side_effects else "0" + if side_effects: + effects_str = f"[yellow]{len(side_effects)}[/yellow]" + + table.add_row(name, complexity_str, effects_str, str(lines)) + + console.print(table) + else: + console.print( + Panel.fit( + f"[yellow]No functions found with complexity >= {threshold}[/yellow]", + title="No Matches", + border_style="yellow", + ) + ) + else: + matches = [ + (name, data) for name, data in functions.items() if query.lower() in name.lower() + ] + + if matches: + console.print( + Panel.fit( + f"[green]Found {len(matches)} functions matching '{query}'[/green]", + title="Search Results", + border_style="green", + ) + ) + + table = Table(title=f"Functions matching '{query}'") + table.add_column("Function", style="cyan", no_wrap=True) + table.add_column("Complexity", justify="center", style="magenta", width=10) + table.add_column("Side Effects", justify="center", style="yellow", width=12) + table.add_column("Lines", justify="center", style="green", width=6) + + for name, data in matches: + complexity = data.get("complexity", 0) + side_effects = data.get("side_effects", []) + lines = data.get("lines", 0) + + complexity_str = str(complexity) + if complexity > 5: + complexity_str = f"[red]{complexity}[/red]" + elif complexity > 3: + complexity_str = f"[yellow]{complexity}[/yellow]" + + effects_str = str(len(side_effects)) if side_effects else "0" + if side_effects: + effects_str = f"[yellow]{len(side_effects)}[/yellow]" + + table.add_row(name, complexity_str, effects_str, str(lines)) + + console.print(table) + else: + console.print( + Panel.fit( + f"[yellow]No functions found matching '{query}'[/yellow]\n\n" + "Try:\n" + "• A different search term\n" + "• A partial function name\n" + "• A complexity threshold (number)", + title="No Matches", + border_style="yellow", + ) + ) + + +def generate_lens_interactive( + console: Console, functions: dict[str, Any], result: dict[str, Any] +) -> None: + """Generate lens interactively.""" + console.print("\n[bold]Generate Understanding Lens[/bold]") + + console.print("Select seed functions (comma-separated numbers):") + func_list = list(functions.items()) + for i, (name, data) in enumerate(func_list[:20]): + complexity = data.get("complexity", 0) + console.print(f" {i + 1:2d}. {name} (complexity: {complexity})") + + if len(func_list) > 20: + console.print(f" ... and {len(func_list) - 20} more") + + selection = typer.prompt("Enter function numbers (e.g., 1,3,5)", default="1,2,3") + + try: + indices = [int(x.strip()) - 1 for x in selection.split(",")] + seeds = [func_list[i][0] for i in indices if 0 <= i < len(func_list)] + + if not seeds: + console.print("[red]No valid functions selected[/red]") + return + + console.print(f"[green]Selected seeds: {seeds}[/green]") + + with console.status("[bold green]Generating lens..."): + lens = lens_from_seeds(seeds, result) + rank_by_error_proximity(lens) + + lens_functions = lens.get("functions", {}) + console.print(f"[green]Generated lens with {len(lens_functions)} functions[/green]") + + console.print("\nLens functions (ranked by importance):") + for i, (name, data) in enumerate(lens_functions.items(), 1): + complexity = data.get("complexity", 0) + console.print(f" {i:2d}. {name} (complexity: {complexity})") + + except (ValueError, IndexError) as e: + console.print(f"[red]Invalid selection: {e}[/red]") + + +def view_function_details(console: Console, functions: dict[str, Any]) -> None: + """View detailed function information.""" + console.print("\n[bold]Function Details[/bold]") + + func_name = typer.prompt("Enter function name") + + if func_name not in functions: + console.print(f"[red]Function '{func_name}' not found[/red]") + return + + data = functions[func_name] + console.print(f"\n[bold]{func_name}[/bold]") + console.print(f" Complexity: {data.get('complexity', 0)}") + console.print(f" Side effects: {', '.join(data.get('side_effects', []) or [])}") + console.print(f" Has docstring: {data.get('has_docstring', 'n/a')}") + console.print(f" Has type hints: {data.get('has_type_hints', 'n/a')}") + callers = data.get("callers") or [] + calls = data.get("calls") or [] + if callers: + console.print(f" Callers: {', '.join(callers)}") + if calls: + console.print(f" Calls: {', '.join(str(c) for c in calls)}") + + +def generate_tour_interactive( + console: Console, functions: dict[str, Any], result: dict[str, Any] +) -> None: + """Generate tour interactively.""" + console.print("\n[bold]Generate Understanding Tour[/bold]") + + seeds_raw = typer.prompt("Enter seed functions (comma-separated)", default="") + + if not seeds_raw: + console.print("[yellow]No seeds provided. Using first 3 functions.[/yellow]") + seeds = list(functions.keys())[:3] + else: + seeds = [s.strip() for s in seeds_raw.split(",")] + valid_seeds = [s for s in seeds if s in functions] + if not valid_seeds: + console.print("[red]No valid seeds found[/red]") + return + seeds = valid_seeds + + console.print(f"[green]Using seeds: {seeds}[/green]") + + with console.status("[bold green]Generating tour..."): + lens = lens_from_seeds(seeds, result) + rank_by_error_proximity(lens) + + if lens.get("functions"): + tour_md = write_tour_md(lens) + tour_path = "understanding_tour.md" + with open(tour_path, "w", encoding="utf-8") as f: + f.write(tour_md) + console.print(f"[green]Tour generated: {tour_path}[/green]") + else: + console.print("[yellow]No functions in lens for tour generation[/yellow]") + + +def export_data_interactive(console: Console, result: dict[str, Any]) -> None: + """Export data interactively.""" + console.print("\n[bold]Export Data[/bold]") + console.print("1. Export as JSON") + console.print("2. Export as Markdown") + console.print("3. Export as SVG") + + choice = typer.prompt("Select format", type=int, default=1) + + if choice == 1: + output_path = "analysis.json" + with open(output_path, "w", encoding="utf-8") as f: + json.dump(result, f, indent=2) + console.print(f"[green]Exported to {output_path}[/green]") + elif choice == 2: + output_path = "analysis.md" + functions = result.get("functions", {}) + with open(output_path, "w", encoding="utf-8") as f: + f.write("# Code Analysis Report\n\n") + f.write(f"Functions analyzed: {len(functions)}\n\n") + f.write("## Functions\n\n") + for name, data in functions.items(): + complexity = data.get("complexity", 0) + side_effects = data.get("side_effects", []) + f.write(f"- **{name}** (complexity: {complexity})\n") + if side_effects: + f.write(f" - Side effects: {', '.join(side_effects)}\n") + console.print(f"[green]Exported to {output_path}[/green]") + elif choice == 3: + console.print("[yellow]SVG export not implemented in TUI mode[/yellow]") + else: + console.print("[red]Invalid choice[/red]") + + +def run_tui(scan_path: str) -> None: + """Launch interactive Text User Interface for code understanding.""" + console = Console() + + try: + console.print( + Panel.fit( + "[bold blue]Understand-First TUI[/bold blue]\n\n" + "Interactive exploration of your codebase.\n" + "Navigate through functions, understand relationships,\n" + "and generate personalized learning paths.", + title="Welcome", + border_style="blue", + ) + ) + console.print("[dim]Press Ctrl+C to exit at any time[/dim]\n") + + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + BarColumn(), + TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), + TimeElapsedColumn(), + console=console, + ) as progress: + load_task = progress.add_task("Loading codebase...", total=100) + progress.update(load_task, advance=30) + time.sleep(0.2) + progress.update(load_task, advance=40) + result = build_repo_map(pathlib.Path(scan_path)) + progress.update(load_task, advance=30) + + functions = result.get("functions", {}) + + if functions: + high_complexity = sum(1 for f in functions.values() if f.get("complexity", 0) > 5) + with_side_effects = sum(1 for f in functions.values() if f.get("side_effects", [])) + + console.print( + Panel.fit( + f"[green]Codebase loaded successfully![/green]\n\n" + f"• [bold]{len(functions)}[/bold] functions available\n" + f"• [bold]{high_complexity}[/bold] high complexity functions\n" + f"• [bold]{with_side_effects}[/bold] functions with side effects", + title="Ready to Explore", + border_style="green", + ) + ) + else: + console.print( + Panel.fit( + "[yellow]No functions found in the specified path[/yellow]\n\n" + "This could mean:\n" + "• The path doesn't contain Python files\n" + "• The files don't have function definitions\n" + "• There's an issue with the file permissions", + title="No Functions Found", + border_style="yellow", + ) + ) + return + + while True: + console.print("\n[bold cyan]Main Menu[/bold cyan]") + + menu_table = Table(show_header=False, box=None) + menu_table.add_column("Option", style="cyan", width=3) + menu_table.add_column("Description", style="white") + + menu_table.add_row("1", "View function list") + menu_table.add_row("2", "Search functions") + menu_table.add_row("3", "Generate understanding lens") + menu_table.add_row("4", "View function details") + menu_table.add_row("5", "Generate tour") + menu_table.add_row("6", "Export data") + menu_table.add_row("7", "View metrics") + menu_table.add_row("0", "Exit") + + console.print(menu_table) + + try: + choice = typer.prompt("\nSelect option", type=int, default=0) + except (ValueError, typer.Abort): + console.print("[red]Invalid input. Please enter a number.[/red]") + continue + + if choice == 0: + console.print( + Panel.fit( + "[yellow]Thanks for using Understand-First TUI![/yellow]\n\n" + "Keep exploring your codebase and building understanding!", + title="Goodbye", + border_style="yellow", + ) + ) + break + elif choice == 1: + show_function_list(console, functions) + elif choice == 2: + search_functions(console, functions) + elif choice == 3: + generate_lens_interactive(console, functions, result) + elif choice == 4: + view_function_details(console, functions) + elif choice == 5: + generate_tour_interactive(console, functions, result) + elif choice == 6: + export_data_interactive(console, result) + elif choice == 7: + show_metrics(console, result) + else: + console.print( + Panel.fit( + "[red]Invalid option. Please try again.[/red]\n\n" + "Select a number between 0-7", + title="Invalid Selection", + border_style="red", + ) + ) + + ttu_record("tui_session", {"functions_loaded": len(functions)}) + + except KeyboardInterrupt: + console.print("\n[yellow]TUI interrupted by user[/yellow]") + console.print("[dim]You can run 'u tui' again anytime to continue[/dim]") + except Exception as e: + console.print( + Panel.fit( + f"[red]Error in TUI:[/red] {str(e)}\n\n" + "This might be due to:\n" + "• File permission issues\n" + "• Corrupted code files\n" + "• Missing dependencies", + title="TUI Error", + border_style="red", + ) + ) + raise typer.Exit(1) from None diff --git a/cli/ucli/commands/init_ops.py b/cli/ucli/commands/init_ops.py new file mode 100644 index 0000000..2d41001 --- /dev/null +++ b/cli/ucli/commands/init_ops.py @@ -0,0 +1,913 @@ +"""Init / wizard / TUI helpers and CLI banner (extracted from main.py).""" + +from __future__ import annotations + +import os +from typing import Any + +import typer +from rich import print +from rich.align import Align +from rich.console import Console +from rich.layout import Layout +from rich.live import Live +from rich.panel import Panel +from rich.table import Table +from rich.text import Text + +from ucli.config import filter_config_to_schema, validate_config_dict + + +def show_banner() -> None: + """Print the Understand-First ASCII banner.""" + console = Console() + banner = """ +[bold cyan] + _ _ _ _ _ _____ _ _ + | | | |_ __ __| | ___ _ __ ___| |_ __ _ _ __ __| | | ___(_)_ __ ___| |_ + | | | | '_ \\ / _` |/ _ \\ '__/ __| __/ _` | '_ \\ / _` | | |_ | | '__/ __| __| + | |_| | | | | (_| | __/ | \\__ \\ || (_| | | | | (_| | | _| | | | \\__ \\ |_ + \\___/|_| |_|\\__,_|\\___|_| |___/\\__\\__,_|_| |_|\\__,_| |_| |_|_| |___/\\__| +[/bold cyan] +""" + console.print(banner) + console.print("[dim]Accelerate code understanding with intelligent analysis[/dim]") + console.print() + + +def show_welcome_message() -> None: + """Print quick-start hints under the banner.""" + console = Console() + table = Table(show_header=False, box=None, padding=(0, 2)) + table.add_column(style="cyan") + table.add_column(style="white") + table.add_row("Quick start:", "u scan . && u lens from-seeds --map maps/repo.json --seed ") + table.add_row("Help:", "u --help | u --help") + console.print(table) + + +def run_init(*, stack: str, ci: str, wizard: bool, tui: bool) -> None: + """Dispatch ``u init`` to TUI, wizard, or basic config write.""" + if tui: + _launch_tui_mode() + elif wizard: + _run_config_wizard() + else: + _create_basic_config(stack, ci) + + +def _create_basic_config(stack: str, ci: str): + """Create a basic configuration file.""" + os.makedirs(".understand-first.yml".replace(".yml", ""), exist_ok=True) # no-op + open(".understand-first.yml", "w", encoding="utf-8").write( + """hops: 2 +seeds: [] +seeds_for: + bug: [examples/app/hot_path.py] +metrics: + enabled: false +""" + ) + os.makedirs("tours", exist_ok=True) + open("README.md", "a", encoding="utf-8").write( + "\n\n## 10-minute tour\nRun `u scan` then `u demo`.\n" + ) + print("[green]Initialized understand-first config and basics[/green]") + + +def _launch_tui_mode(): + """Launch interactive TUI mode for understand-first.""" + Console() + + # Create layout + layout = Layout() + layout.split_column( + Layout(name="header", size=3), Layout(name="main"), Layout(name="footer", size=3) + ) + + layout["main"].split_row(Layout(name="sidebar", size=30), Layout(name="content")) + + # Header + header_panel = Panel( + Align.center("[bold blue]🧠 Understand-First TUI[/bold blue]"), style="blue" + ) + layout["header"].update(header_panel) + + # Footer + footer_text = Text("Press 'q' to quit, 'h' for help, arrow keys to navigate", style="dim") + layout["footer"].update(Panel(Align.center(footer_text), style="dim")) + + # Main TUI loop + with Live(layout, refresh_per_second=4, screen=True) as live: + _run_tui_main_loop(live, layout) + + +def _run_tui_main_loop(live, layout): + """Main TUI event loop.""" + console = Console() + current_view = "dashboard" + + while True: + # Update content based on current view + if current_view == "dashboard": + _render_dashboard(layout) + elif current_view == "scan": + _render_scan_view(layout) + elif current_view == "lens": + _render_lens_view(layout) + elif current_view == "tour": + _render_tour_view(layout) + elif current_view == "config": + _render_config_view(layout) + + # Handle input + try: + key = console.input() + if key.lower() == "q": + break + elif key.lower() == "h": + _show_help(layout) + elif key == "1": + current_view = "dashboard" + elif key == "2": + current_view = "scan" + elif key == "3": + current_view = "lens" + elif key == "4": + current_view = "tour" + elif key == "5": + current_view = "config" + except KeyboardInterrupt: + break + + +def _render_dashboard(layout): + """Render the main dashboard view.""" + # Sidebar + sidebar_content = """ +[bold]Navigation[/bold] +1. Dashboard +2. Scan Code +3. Lens Management +4. Tour Builder +5. Configuration + +[bold]Quick Actions[/bold] +• Press 's' to scan current directory +• Press 'l' to create new lens +• Press 't' to generate tour +• Press 'c' to configure project +""" + layout["sidebar"].update(Panel(sidebar_content, title="Menu", border_style="blue")) + + # Main content + dashboard_content = """ +[bold green]Welcome to Understand-First TUI![/bold green] + +This interactive interface helps you: +• Analyze your codebase structure +• Create focused lenses for specific areas +• Generate understanding tours +• Track Time-to-Understanding metrics + +[bold]Recent Activity[/bold] +• No recent scans found +• No active lenses +• No tours generated + +[bold]Project Status[/bold] +• Configuration: Not initialized +• Last scan: Never +• Active lenses: 0 +• Generated tours: 0 + +[bold]Quick Start[/bold] +1. Press 'c' to configure your project +2. Press 's' to scan your codebase +3. Press 'l' to create your first lens +4. Press 't' to generate a tour +""" + layout["content"].update(Panel(dashboard_content, title="Dashboard", border_style="green")) + + +def _render_scan_view(layout): + """Render the scan view.""" + # Sidebar + sidebar_content = """ +[bold]Scan Options[/bold] +• Target: Current directory +• Output: maps/out.json +• Verbose: False +• Debug: False + +[bold]Analysis Types[/bold] +• Python: AST analysis +• Dependencies: Import tracking +• Side effects: I/O detection +• Complexity: Cyclomatic metrics +""" + layout["sidebar"].update(Panel(sidebar_content, title="Scan Config", border_style="yellow")) + + # Main content + scan_content = """ +[bold yellow]Code Analysis Scanner[/bold yellow] + +This will analyze your codebase and generate: +• Function dependency graph +• Side effect analysis +• Complexity metrics +• Hot path identification + +[bold]Ready to scan?[/bold] +Press 'Enter' to start analysis or 'Esc' to go back. + +[dim]Note: Large codebases may take several minutes to analyze.[/dim] +""" + layout["content"].update(Panel(scan_content, title="Scan Code", border_style="yellow")) + + +def _render_lens_view(layout): + """Render the lens management view.""" + # Sidebar + sidebar_content = """ +[bold]Lens Types[/bold] +• Entry Points +• Hot Paths +• Side Effects +• High Complexity +• Custom Seeds + +[bold]Presets[/bold] +• bug: Error-related functions +• feature: New feature code +• api: API endpoints +• test: Test functions +""" + layout["sidebar"].update(Panel(sidebar_content, title="Lens Types", border_style="cyan")) + + # Main content + lens_content = """ +[bold cyan]Lens Management[/bold cyan] + +Lenses help you focus on specific parts of your codebase: +• Filter by function type +• Highlight important paths +• Isolate side effects +• Focus on complexity hotspots + +[bold]Create New Lens[/bold] +1. Choose lens type +2. Set seed functions +3. Configure filters +4. Save as preset + +[bold]Existing Lenses[/bold] +• No lenses created yet +""" + layout["content"].update(Panel(lens_content, title="Lens Management", border_style="cyan")) + + +def _render_tour_view(layout): + """Render the tour builder view.""" + # Sidebar + sidebar_content = """ +[bold]Tour Steps[/bold] +1. Introduction +2. Architecture Overview +3. Key Components +4. Data Flow +5. Side Effects +6. Testing Strategy + +[bold]Export Options[/bold] +• Markdown +• HTML +• PDF +• Interactive Web +""" + layout["sidebar"].update(Panel(sidebar_content, title="Tour Builder", border_style="magenta")) + + # Main content + tour_content = """ +[bold magenta]Tour Builder[/bold magenta] + +Generate interactive tours to help team members understand your codebase: +• Step-by-step walkthrough +• Code highlighting +• Interactive navigation +• Export for documentation + +[bold]Create New Tour[/bold] +1. Select starting functions +2. Define tour steps +3. Add explanations +4. Generate and export + +[bold]Tour Templates[/bold] +• Onboarding tour +• Architecture overview +• Feature walkthrough +• Debugging guide +""" + layout["content"].update(Panel(tour_content, title="Tour Builder", border_style="magenta")) + + +def _render_config_view(layout): + """Render the configuration view.""" + # Sidebar + sidebar_content = """ +[bold]Config Sections[/bold] +• Project Type +• Analysis Settings +• Seeds & Presets +• Metrics & Tracking +• CI Integration +• IDE Integration + +[bold]Current Status[/bold] +• Project: Not configured +• Analysis: Default settings +• Seeds: None defined +• Metrics: Disabled +""" + layout["sidebar"].update(Panel(sidebar_content, title="Configuration", border_style="red")) + + # Main content + config_content = """ +[bold red]Project Configuration[/bold red] + +Configure understand-first for your specific project: +• Set analysis depth and scope +• Define seed functions +• Configure metrics tracking +• Set up CI/CD integration + +[bold]Configuration Wizard[/bold] +Press 'w' to run the interactive wizard +or manually edit .understand-first.yml + +[bold]Current Settings[/bold] +• Hops: 2 +• Seeds: [] +• Metrics: Disabled +• CI: Not configured +""" + layout["content"].update(Panel(config_content, title="Configuration", border_style="red")) + + +def _show_help(layout): + """Show help information.""" + help_content = """ +[bold]Keyboard Shortcuts[/bold] + +Navigation: +• 1-5: Switch between views +• q: Quit application +• h: Show this help + +Actions: +• s: Start code scan +• l: Create new lens +• t: Generate tour +• c: Configure project +• w: Run configuration wizard + +General: +• Enter: Confirm action +• Esc: Go back +• Ctrl+C: Force quit +""" + layout["content"].update(Panel(help_content, title="Help", border_style="blue")) + + +def _run_config_wizard(): + """Run interactive configuration wizard with enhanced features.""" + print("[bold blue]🧠 Understand-First Configuration Wizard[/bold blue]") + print("This wizard will help you set up your .understand-first.yml configuration.") + print("The wizard will guide you through project-specific optimizations and best practices.") + print() + + # Enhanced project type selection + project_types = { + "1": ("python", "Python project", "General Python applications and libraries"), + "2": ( + "django", + "Django web application", + "Django web apps with models, views, and templates", + ), + "3": ("fastapi", "FastAPI web application", "Modern async API applications with FastAPI"), + "4": ("flask", "Flask web application", "Flask web apps with blueprints and extensions"), + "5": ( + "microservices", + "Microservices architecture", + "Distributed systems with multiple services", + ), + "6": ("react", "React frontend", "React applications with components and hooks"), + "7": ( + "nodejs", + "Node.js application", + "Node.js applications with Express or other frameworks", + ), + "8": ("go", "Go application", "Go applications and microservices"), + "9": ("java", "Java application", "Java applications with Spring or other frameworks"), + "10": ("custom", "Custom configuration", "Manually configure all settings"), + } + + print("What type of project are you configuring?") + for key, (_value, description, details) in project_types.items(): + print(f" {key}. {description}") + print(f" {details}") + print() + + while True: + choice = typer.prompt("Enter your choice (1-10)", type=str) + if choice in project_types: + project_type, project_name, project_details = project_types[choice] + break + print("[red]Invalid choice. Please enter 1-10.[/red]") + + print(f"\n[green]Selected:[/green] {project_name}") + print(f"[dim]{project_details}[/dim]") + + # Load template if available + template_config = _load_project_template(project_type) + + # Only keys honored by runtime SCHEMA (see ucli.config.SCHEMA). + config = { + "hops": 2, + "seeds": [], + "seeds_for": {}, + "contracts_paths": [], + "glossary_path": "docs/glossary.md", + "metrics": {"enabled": False}, + } + + # Merge template configuration (schema-filtered) + if template_config: + config.update(filter_config_to_schema(template_config)) + print(f"\n[green]Loaded template configuration for {project_name}[/green]") + + # Hops configuration with better validation + print("\n[bold]Analysis Depth Configuration[/bold]") + print("Hops determine how deep the analysis should traverse from seed functions.") + print("Higher values provide more comprehensive analysis but may be slower.") + print("[dim]Schema allows hops in 0..5.[/dim]") + + hops = typer.prompt( + "How many hops should the lens traverse? (0-5, default: 2)", + type=int, + default=config.get("hops", 2), + ) + config["hops"] = max(0, min(5, hops)) + + # Enhanced seeds configuration + print("\n[bold]Seeds Configuration[/bold]") + print("Seeds are starting points for understanding analysis.") + print("You can add files, functions, modules, or patterns as seeds.") + print("Examples: 'main.py', 'app.py:main', '*/models.py', 'service.*'") + + add_seeds = typer.confirm( + "Would you like to add custom seeds?", default=len(config.get("seeds", [])) == 0 + ) + if add_seeds: + seeds = config.get("seeds", []) + print("\n[dim]Enter seeds one by one. Press Enter with empty input to finish.[/dim]") + while True: + seed = typer.prompt( + "Enter a seed (file path, function, or pattern) or press Enter to finish", + default="", + ) + if not seed: + break + if seed not in seeds: + seeds.append(seed) + print(f"[green]Added seed:[/green] {seed}") + else: + print(f"[yellow]Seed already exists:[/yellow] {seed}") + config["seeds"] = seeds + + # Enhanced preset seeds configuration + print("\n[bold]Preset Seeds for Common Scenarios[/bold]") + print("Define preset seed collections for common development scenarios.") + print("These can be used with commands like 'u lens preset bug' or 'u lens preset feature'.") + + add_presets = typer.confirm( + "Would you like to configure preset seed collections?", default=True + ) + if add_presets: + presets = config.get("seeds_for", {}) + + # Suggest common presets based on project type + suggested_presets = _get_suggested_presets(project_type) + if suggested_presets: + print(f"\n[bold]Suggested presets for {project_name}:[/bold]") + for preset_name, preset_seeds in suggested_presets.items(): + print(f" • {preset_name}: {', '.join(preset_seeds)}") + + use_suggested = typer.confirm("Use these suggested presets?", default=True) + if use_suggested: + presets.update(suggested_presets) + + # Allow custom presets + add_custom_presets = typer.confirm("Add custom presets?", default=False) + if add_custom_presets: + while True: + preset_name = typer.prompt( + "Enter preset name (e.g., 'bug', 'feature', 'api') or press Enter to finish", + default="", + ) + if not preset_name: + break + + preset_seeds = [] + print(f"\n[dim]Enter seeds for '{preset_name}' preset:[/dim]") + while True: + seed = typer.prompt( + f"Enter seed for '{preset_name}' preset or press Enter to finish", + default="", + ) + if not seed: + break + preset_seeds.append(seed) + + if preset_seeds: + presets[preset_name] = preset_seeds + print( + f"[green]Added preset '{preset_name}' with {len(preset_seeds)} seeds[/green]" + ) + + config["seeds_for"] = presets + + # Enhanced contract configuration + print("\n[bold]Contract Configuration[/bold]") + print("Contracts define API specifications and formal verification requirements.") + print("These help ensure API compliance and can generate property tests.") + + add_contracts = typer.confirm("Do you have contract files to include?", default=False) + if add_contracts: + contract_paths = config.get("contracts_paths", []) + print("\n[dim]Enter contract file paths one by one:[/dim]") + while True: + path = typer.prompt("Enter contract file path or press Enter to finish", default="") + if not path: + break + if os.path.exists(path): + contract_paths.append(path) + print(f"[green]Added contract:[/green] {path}") + else: + print(f"[yellow]File not found:[/yellow] {path}") + add_anyway = typer.confirm("Add anyway?", default=False) + if add_anyway: + contract_paths.append(path) + config["contracts_paths"] = contract_paths + + # Enhanced metrics configuration + print("\n[bold]Metrics and Analytics Configuration[/bold]") + print("Understand-First can track Time To Understanding (TTU) metrics and generate reports.") + + enable_metrics = typer.confirm("Enable TTU metrics tracking?", default=False) + config["metrics"] = {"enabled": bool(enable_metrics)} + + # Write configuration with validation (SCHEMA keys only) + try: + import yaml + + config = filter_config_to_schema(config) + errors = validate_config_dict(config) + if errors: + print("\n[red]Configuration validation errors:[/red]") + for error in errors: + print(f" • {error}") + + fix_errors = typer.confirm("Fix errors automatically?", default=True) + if fix_errors: + config = filter_config_to_schema(_fix_config_errors(config, errors)) + else: + print("[yellow]Saving configuration with errors. Please fix manually.[/yellow]") + + config_yaml = yaml.dump(config, default_flow_style=False, sort_keys=False, indent=2) + + with open(".understand-first.yml", "w", encoding="utf-8") as f: + f.write(config_yaml) + + # Create necessary directories + directories = ["tours", "maps", "traces", "contracts", "docs", "fixtures"] + for directory in directories: + os.makedirs(directory, exist_ok=True) + + # Create example files + _create_example_files(project_type) + + # Update README if it exists + _update_readme_with_integration(project_type) + + print("\n[green]✅ Configuration saved to .understand-first.yml[/green]") + print("\n[bold]Next steps:[/bold]") + print("1. Run `u scan . -o maps/repo.json` to generate a repository map") + print("2. Run `u demo` for a guided demonstration") + print("3. Run `u doctor` to verify your setup") + print("4. Run `u config-validate` to confirm the config schema") + + print("\n[dim]Configuration wizard completed successfully![/dim]") + + except Exception as e: + print(f"\n[red]Error saving configuration:[/red] {e}") + raise typer.Exit(1) from None + + +def _load_project_template(project_type: str) -> dict[str, Any] | None: + """Load project template configuration if available (SCHEMA keys only).""" + template_path = f"templates/{project_type}/.understand-first.yml" + + if os.path.exists(template_path): + try: + import yaml + + with open(template_path, encoding="utf-8") as f: + data = yaml.safe_load(f) or {} + if isinstance(data, dict): + return filter_config_to_schema(data) + except Exception as e: + print(f"Warning: Failed to load template {template_path}: {e}") + + return None + + +def _get_suggested_presets(project_type: str) -> dict[str, list[str]]: + """Get suggested presets based on project type.""" + presets = { + "python": { + "main": ["main.py", "app.py", "run.py"], + "tests": ["test_*.py", "tests/*.py"], + "utils": ["utils/*.py", "helpers/*.py"], + }, + "django": { + "models": ["*/models.py"], + "views": ["*/views.py"], + "urls": ["*/urls.py"], + "admin": ["*/admin.py"], + "forms": ["*/forms.py"], + "tests": ["*/test*.py", "*/tests.py"], + }, + "fastapi": { + "routes": ["*/routes/*.py", "*/api/*.py", "main.py"], + "models": ["*/models.py", "*/schemas.py"], + "services": ["*/services/*.py", "*/core/*.py"], + "tests": ["test_*.py", "*/test_*.py"], + }, + "flask": { + "routes": ["app.py", "*/routes/*.py", "*/blueprints/*.py"], + "models": ["*/models.py"], + "forms": ["*/forms.py"], + "templates": ["*/templates/**/*.html"], + "tests": ["test_*.py", "*/test_*.py"], + }, + "microservices": { + "services": ["*/service*.py", "*/api*.py"], + "clients": ["*/client*.py"], + "models": ["*/models.py", "*/schemas.py"], + "tests": ["*/test*.py", "*/tests.py"], + }, + "react": { + "components": ["src/components/**/*.js", "src/components/**/*.jsx"], + "pages": ["src/pages/**/*.js", "src/pages/**/*.jsx"], + "hooks": ["src/hooks/**/*.js"], + "utils": ["src/utils/**/*.js", "src/helpers/**/*.js"], + "tests": ["src/**/*.test.js", "src/**/*.spec.js"], + }, + "nodejs": { + "routes": ["routes/*.js", "*/routes/*.js"], + "controllers": ["controllers/*.js", "*/controllers/*.js"], + "models": ["models/*.js", "*/models/*.js"], + "middleware": ["middleware/*.js", "*/middleware/*.js"], + "tests": ["test/*.js", "*/test/*.js", "**/*.test.js"], + }, + "go": { + "main": ["main.go", "cmd/**/*.go"], + "handlers": ["handlers/*.go", "*/handlers/*.go"], + "models": ["models/*.go", "*/models/*.go"], + "services": ["services/*.go", "*/services/*.go"], + "tests": ["*_test.go", "**/*_test.go"], + }, + "java": { + "controllers": ["**/controller/*.java", "**/web/*.java"], + "services": ["**/service/*.java", "**/business/*.java"], + "models": ["**/model/*.java", "**/entity/*.java"], + "repositories": ["**/repository/*.java", "**/dao/*.java"], + "tests": ["**/test/**/*.java", "**/*Test.java"], + }, + } + + return presets.get(project_type, {}) + + +def _fix_config_errors(config: dict[str, Any], errors: list[str]) -> dict[str, Any]: + """Fix common configuration errors automatically (SCHEMA-aligned).""" + fixed_config = filter_config_to_schema(config) + + for error in errors: + if "hops" in error: + fixed_config["hops"] = max(0, min(5, int(fixed_config.get("hops", 2) or 2))) + elif "seeds_for" in error: + fixed_config["seeds_for"] = {} + elif "metrics" in error: + fixed_config["metrics"] = {"enabled": False} + elif "contracts_paths" in error: + fixed_config["contracts_paths"] = [] + elif "seeds" in error: + fixed_config["seeds"] = [] + elif "glossary_path" in error: + fixed_config["glossary_path"] = "docs/glossary.md" + + return filter_config_to_schema(fixed_config) + + +def _create_example_files(project_type: str) -> None: + """Create example files based on project type.""" + try: + # Create example tour + tour_content = f"""# Understanding Tour Example + +This is an example tour generated for a {project_type} project. + +## Getting Started + +1. Run `u scan . -o maps/repo.json` to generate a repository map +2. Run `u lens from-seeds --map maps/repo.json --seed main -o maps/lens.json` +3. Run `u tour maps/lens.json -o tours/understanding.md` + +## Project-Specific Tips + +For {project_type} projects, focus on: +- Main entry points and application structure +- Key business logic and data models +- API endpoints and routing +- Test coverage and quality + +## Next Steps + +- Add more seeds based on your specific use cases +- Configure CI integration for automated analysis +- Set up IDE integration for real-time insights +""" + + with open("tours/example.md", "w", encoding="utf-8") as f: + f.write(tour_content) + + # Create example fixture + fixture_content = f"""# Example Fixture for {project_type} Project + +This fixture demonstrates how to test the understanding generated by Understand-First. + +## Usage + +Run this fixture to verify that the understanding tour can be executed: + +```bash +u tour_run --fixtures fixtures maps/lens.json +``` + +## Customization + +Modify this fixture to match your project's specific requirements and test scenarios. +""" + + with open("fixtures/example_fixture.py", "w", encoding="utf-8") as f: + f.write(fixture_content) + + # Create example contract + contract_content = f"""# Example Contract for {project_type} Project + +This is an example contract file that defines API specifications and formal verification requirements. + +## Contract Definition + +```yaml +# Example API contract +ROUTE::api: + GET /health: + request_schema: {{}} + response_schema: + type: object + properties: + status: + type: string + enum: [healthy, unhealthy] + timestamp: + type: string + format: date-time + preconditions: [] + postconditions: ["response.status_code == 200"] + side_effects: [] +``` + +## Usage + +1. Define your API contracts in this file +2. Run `u contracts verify` to check compliance +3. Generate property tests with `u contracts stub-tests` +""" + + with open("contracts/example_contracts.yaml", "w", encoding="utf-8") as f: + f.write(contract_content) + + print("[green]Created example files:[/green]") + print(" • tours/example.md") + print(" • fixtures/example_fixture.py") + print(" • contracts/example_contracts.yaml") + + except Exception as e: + print(f"Warning: Failed to create example files: {e}") + + +def _update_readme_with_integration(project_type: str) -> None: + """Update README with Understand-First integration section.""" + if not os.path.exists("README.md"): + return + + try: + integration_section = f""" + +## Understand-First Integration + +This project uses [Understand-First](https://github.com/SentinelOps-CI/understand-first) for automated code understanding and documentation generation. + +### Quick Start + +1. **Generate Repository Map** + ```bash + u scan . -o maps/repo.json + ``` + +2. **Create Understanding Lens** + ```bash + u lens from-seeds --map maps/repo.json --seed main -o maps/lens.json + ``` + +3. **Generate Understanding Tour** + ```bash + u tour maps/lens.json -o tours/understanding.md + ``` + +4. **Run Guided Demo** + ```bash + u demo + ``` + +### Project-Specific Configuration + +This {project_type} project is configured with optimized settings for: +- **Seeds**: Key entry points and important modules +- **Analysis**: Complexity analysis, side effect detection, and dependency tracking +- **Integration**: IDE support and CI/CD pipeline integration + +### Understanding Features + +- **Interactive Tours**: Step-by-step walkthroughs of complex code paths +- **Runtime Tracing**: Actual execution paths, not just static analysis +- **Contract Verification**: API compliance and formal verification +- **Metrics Tracking**: Time To Understanding (TTU) measurement +- **IDE Integration**: Real-time insights in your development environment + +### Configuration + +The project configuration is stored in `.understand-first.yml`. Key settings include: + +- **Hops**: Analysis depth (currently set to {2}) +- **Seeds**: Starting points for analysis +- **Presets**: Common scenarios like bug fixes and feature development +- **Patterns**: File inclusion/exclusion rules + +### CI/CD Integration + +The project is configured for CI/CD integration. Add this to your pipeline: + +```yaml +- name: Understand-First Analysis + run: | + u scan . -o maps/repo.json + u lens preset feature --map maps/repo.json -o maps/lens.json + u tour maps/lens.json -o tours/ci-tour.md +``` + +### IDE Integration + +Install the Understand-First VS Code extension for: +- Gutter annotations showing complexity and call counts +- Quick peek tours and explanations +- Real-time understanding insights + +### Learn More + +- [Documentation](https://github.com/SentinelOps-CI/understand-first#readme) +- [Examples](https://github.com/SentinelOps-CI/understand-first/tree/main/examples) +- [Web Demo](https://sentinelops-ci.github.io/understand-first/demo) +""" + + with open("README.md", "a", encoding="utf-8") as f: + f.write(integration_section) + + print("[green]Updated README.md with Understand-First integration section[/green]") + + except Exception as e: + print(f"Warning: Failed to update README: {e}") diff --git a/cli/ucli/commands/lens_ops.py b/cli/ucli/commands/lens_ops.py new file mode 100644 index 0000000..42fa069 --- /dev/null +++ b/cli/ucli/commands/lens_ops.py @@ -0,0 +1,255 @@ +"""Lens command implementations (extracted from main.py).""" + +from __future__ import annotations + +import json +import os +import pathlib +import traceback +from typing import Any + +import typer +from rich.console import Console +from rich.panel import Panel +from rich.progress import ( + BarColumn, + Progress, + SpinnerColumn, + TextColumn, + TimeElapsedColumn, +) +from rich.prompt import Prompt +from rich.table import Table + +from ucli.config import load_config, load_preset +from ucli.lens.ingest import seeds_from_github_log, seeds_from_jira +from ucli.lens.lens import ( + explain_node, + lens_from_issue, + lens_from_seeds, + merge_trace_into_lens, + rank_by_error_proximity, +) +from ucli.metrics.ttu import record as ttu_record + + +def run_lens_from_issue(issue_md: str, map_path: str, o: str) -> None: + with open(map_path, encoding="utf-8") as f: + repo_map = json.load(f) + lens = lens_from_issue(issue_md, repo_map) + rank_by_error_proximity(lens) + os.makedirs(pathlib.Path(o).parent, exist_ok=True) + with open(o, "w", encoding="utf-8") as f: + json.dump(lens, f, indent=2) + from rich import print as rprint + + rprint(f"[green]Wrote[/green] {o}") + + +def run_lens_from_seeds( + seed: list[str], + map_path: str, + o: str, + *, + interactive: bool = False, + verbose: bool = False, + exact: bool = False, +) -> None: + """Create understanding lens from seed functions with enhanced TUI.""" + console = Console() + + try: + with open(map_path, encoding="utf-8") as f: + repo_map = json.load(f) + + seeds = seed.copy() if seed else [] + + if interactive: + console.print( + Panel( + "[bold blue]🔍 Interactive Seed Selection[/bold blue]\n" + "Select functions to include in your understanding lens.\n" + "This will help focus your analysis on specific areas of the codebase.", + title="Lens Creation", + border_style="blue", + ) + ) + + functions = list(repo_map.get("functions", {}).keys()) + if not functions: + console.print("[red]No functions found in map. Run 'u scan' first.[/red]") + raise typer.Exit(1) from None + + func_table = Table(title="Available Functions", show_header=True) + func_table.add_column("Index", style="cyan", width=6) + func_table.add_column("Function", style="white") + func_table.add_column("File", style="dim") + func_table.add_column("Complexity", style="yellow") + + for i, func_name in enumerate(functions, 1): + func_data = repo_map["functions"][func_name] + complexity = func_data.get("complexity", 0) + file_path = func_data.get("file", "unknown") + func_table.add_row(str(i), func_name, file_path, str(complexity)) + + console.print(func_table) + console.print() + + while True: + choice = Prompt.ask("Select function index (or 'done' to finish)", default="done") + + if choice.lower() == "done": + break + + try: + index = int(choice) - 1 + if 0 <= index < len(functions): + selected_func = functions[index] + if selected_func not in seeds: + seeds.append(selected_func) + console.print(f"[green]✓ Added[/green] {selected_func}") + else: + console.print(f"[yellow]⚠ Already selected[/yellow] {selected_func}") + else: + console.print("[red]Invalid index. Please try again.[/red]") + except ValueError: + console.print("[red]Please enter a valid number or 'done'.[/red]") + + if not seeds: + console.print("[yellow]No seeds selected. Using first 5 functions.[/yellow]") + seeds = functions[:5] + + config_table = Table(title="Lens Configuration", show_header=False) + config_table.add_column("Setting", style="cyan") + config_table.add_column("Value", style="white") + config_table.add_row("Map File", map_path) + config_table.add_row("Seeds", f"{len(seeds)} selected") + config_table.add_row("Output File", o) + + console.print(config_table) + console.print() + + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + BarColumn(), + TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), + TimeElapsedColumn(), + console=console, + ) as progress: + lens_task = progress.add_task("Creating understanding lens...", total=100) + progress.update(lens_task, advance=30) + + lens = lens_from_seeds(seeds, repo_map, exact=exact) + rank_by_error_proximity(lens) + + progress.update(lens_task, advance=70) + + os.makedirs(pathlib.Path(o).parent, exist_ok=True) + with open(o, "w", encoding="utf-8") as f: + json.dump(lens, f, indent=2) + + progress.update(lens_task, advance=100) + + lens_functions = len(lens.get("functions", {})) + + results_panel = Panel( + f"[green]✓ Lens created successfully![/green]\n\n" + f"📊 [bold]Lens Summary:[/bold]\n" + f" • Functions in lens: {lens_functions}\n" + f" • Seeds used: {len(seeds)}\n" + f" • Output file: {o}\n\n" + f"🎯 [bold]Next steps:[/bold]\n" + f" • Run [cyan]u tour {o}[/cyan] to generate an understanding tour\n" + f" • Run [cyan]u trace module [/cyan] to add runtime traces", + title="Lens Creation Results", + border_style="green", + ) + + console.print(results_panel) + + ttu_record( + "lens_created", + { + "seeds_count": len(seeds), + "lens_functions": lens_functions, + "interactive": interactive, + }, + ) + + except Exception as e: + console.print(f"[red]Error creating lens:[/red] {str(e)}") + if verbose: + console.print(f"[dim]{traceback.format_exc()}[/dim]") + raise typer.Exit(1) from None + + +def run_lens_merge_trace(lens_json: str, trace_json: str, o: str) -> None: + with open(lens_json, encoding="utf-8") as f: + lens = json.load(f) + with open(trace_json, encoding="utf-8") as f: + trace = json.load(f) + merged = merge_trace_into_lens(lens, trace) + rank_by_error_proximity(merged) + os.makedirs(pathlib.Path(o).parent, exist_ok=True) + with open(o, "w", encoding="utf-8") as f: + json.dump(merged, f, indent=2) + from rich import print as rprint + + rprint(f"[green]Wrote[/green] {o}") + + +def run_lens_preset(label: str, map_path: str, o: str) -> None: + cfg = load_config() + seeds = load_preset(label) + hops = cfg.get("hops", 2) + with open(map_path, encoding="utf-8") as f: + repo_map = json.load(f) + lens = lens_from_seeds(seeds, repo_map, hops=hops) + with open(o, "w", encoding="utf-8") as wf: + json.dump(lens, wf, indent=2) + from rich import print as rprint + + rprint(f"[green]Lens written[/green] {o}") + + +def run_ingest_github(log_path: str) -> None: + seeds = seeds_from_github_log(log_path) + from rich import print as rprint + + rprint(json.dumps({"seeds": seeds}, indent=2)) + + +def run_ingest_jira(jira_path: str) -> None: + seeds = seeds_from_jira(jira_path) + from rich import print as rprint + + rprint(json.dumps({"seeds": seeds}, indent=2)) + + +def run_lens_explain( + qname: str, + lens: str, + repo: str, + *, + json_out: bool = False, +) -> None: + with open(lens, encoding="utf-8") as f: + lens_data = json.load(f) + with open(repo, encoding="utf-8") as f: + repo_map = json.load(f) + info = explain_node(qname, lens_data, repo_map) + from rich import print as rprint + + if json_out: + rprint(json.dumps(info, indent=2)) + return + rprint(info.get("qname")) + rprint(" reason:") + for r in info.get("reason", []): + k, v = next(iter(r.items())) + rprint(f" - {k}: {v}") + edges: dict[str, Any] = info.get("edges", {}) + rprint(" edges:") + rprint(" callers:", ", ".join(edges.get("callers", []))) + rprint(" callees:", ", ".join(edges.get("callees", []))) diff --git a/cli/ucli/commands/metrics_ops.py b/cli/ucli/commands/metrics_ops.py new file mode 100644 index 0000000..3d521e7 --- /dev/null +++ b/cli/ucli/commands/metrics_ops.py @@ -0,0 +1,226 @@ +"""Metrics display helpers and `u metrics` body (extracted from main.py).""" + +from __future__ import annotations + +import json +from datetime import datetime +from typing import Any + +import typer +from rich.console import Console +from rich.panel import Panel +from rich.table import Table + +from ucli.metrics.analytics import get_dashboard_data, get_tracker, track_event + + +def show_metrics_dashboard(console: Console, data: dict[str, Any]): + """Display comprehensive metrics dashboard""" + + # Header + console.print( + Panel( + "[bold blue]📊 Understand-First Metrics Dashboard[/bold blue]\n" + f"Period: Last {data['period_days']} days\n" + f"Generated: {datetime.fromtimestamp(data['timestamp']).strftime('%Y-%m-%d %H:%M:%S')}", + title="Dashboard", + border_style="blue", + ) + ) + + # North Star Goals Status + goals_table = Table(title="North Star Goals Status", show_header=True) + goals_table.add_column("Goal", style="cyan") + goals_table.add_column("Target", style="yellow") + goals_table.add_column("Current", style="white") + goals_table.add_column("Status", style="green") + + # TTU Goal + ttu_metrics = data["ttu_metrics"] + ttu_current = ttu_metrics["average_ttu_minutes"] + ttu_target = data["north_star_goals"]["ttu_target"] + ttu_status = "✅" if ttu_current and ttu_current <= ttu_target else "❌" + goals_table.add_row( + "TTU (minutes)", + f"≤{ttu_target}", + f"{ttu_current:.1f}" if ttu_current else "N/A", + ttu_status, + ) + + # Activation Goal + activation_metrics = data["activation_metrics"] + activation_current = activation_metrics["activation_rate_percentage"] + activation_target = data["north_star_goals"]["activation_target"] + activation_status = "✅" if activation_current >= activation_target else "❌" + goals_table.add_row( + "Activation Rate (%)", + f"≥{activation_target}", + f"{activation_current:.1f}", + activation_status, + ) + + # Tour Completion Goal + tour_metrics = data["tour_completion_metrics"] + tour_current = tour_metrics["completion_rate_percentage"] + tour_target = data["north_star_goals"]["tour_completion_target"] + tour_status = "✅" if tour_current >= tour_target else "❌" + goals_table.add_row( + "Tour Completion (%)", f"≥{tour_target}", f"{tour_current:.1f}", tour_status + ) + + # PR Coverage Goal + pr_metrics = data["pr_coverage_metrics"] + pr_current = pr_metrics["coverage_rate_percentage"] + pr_target = data["north_star_goals"]["pr_coverage_target"] + pr_status = "✅" if pr_current >= pr_target else "❌" + goals_table.add_row("PR Coverage (%)", f"≥{pr_target}", f"{pr_current:.1f}", pr_status) + + console.print(goals_table) + console.print() + + # Detailed Metrics + # TTU Metrics + ttu_panel = Panel( + f"📈 **TTU Metrics**\n" + f"• Total sessions: {ttu_metrics['total_sessions']}\n" + f"• Sessions with tour completion: {ttu_metrics['sessions_with_tour_completion']}\n" + f"• Average TTU: {ttu_metrics['average_ttu_minutes']:.1f} minutes\n" + f"• Under 10 minutes: {ttu_metrics['ttu_under_10_min_percentage']:.1f}%", + title="Time-to-Understanding", + border_style="blue", + ) + console.print(ttu_panel) + + # Activation Metrics + activation_panel = Panel( + f"🚀 **Activation Metrics**\n" + f"• Total sessions: {activation_metrics['total_sessions']}\n" + f"• Activated sessions: {activation_metrics['activated_sessions']}\n" + f"• Activation rate: {activation_metrics['activation_rate_percentage']:.1f}%\n" + f"• Under 2 minutes: {activation_metrics['under_2_min_percentage']:.1f}%", + title="User Activation", + border_style="green", + ) + console.print(activation_panel) + + # Tour Completion Metrics + tour_panel = Panel( + f"📚 **Tour Completion Metrics**\n" + f"• Total tours: {tour_metrics['total_tours']}\n" + f"• Completed tours: {tour_metrics['completed_tours']}\n" + f"• Completion rate: {tour_metrics['completion_rate_percentage']:.1f}%\n" + f"• Average completion: {tour_metrics['average_completion_percentage']:.1f}%", + title="Tour Completion", + border_style="yellow", + ) + console.print(tour_panel) + + # PR Coverage Metrics + pr_panel = Panel( + f"🔀 **PR Coverage Metrics**\n" + f"• Total PRs: {pr_metrics['total_prs']}\n" + f"• PRs with artifacts: {pr_metrics['prs_with_artifacts']}\n" + f"• Coverage rate: {pr_metrics['coverage_rate_percentage']:.1f}%", + title="PR Coverage", + border_style="red", + ) + console.print(pr_panel) + + +def run_metrics( + *, + dashboard: bool = False, + days: int = 30, + export: str | None = None, + track: str | None = None, +) -> None: + """View and manage Understand-First metrics for TTU/TTFSC goals.""" + console = Console() + try: + if track: + session_id = get_tracker().start_session() + track_event(track, session_id) + console.print(f"[green]Tracked event: {track}[/green]") + return + + dashboard_data = get_dashboard_data(days) + + if dashboard: + show_metrics_dashboard(console, dashboard_data) + else: + show_metrics_summary(console, dashboard_data) + + if export: + with open(export, "w", encoding="utf-8") as f: + json.dump(dashboard_data, f, indent=2) + console.print(f"[green]Metrics exported to {export}[/green]") + except Exception as e: + console.print(f"[red]Error with metrics:[/red] {str(e)}") + raise typer.Exit(1) from None + + +def show_metrics_summary(console: Console, data: dict[str, Any]): + """Display metrics summary""" + + # Quick summary table + summary_table = Table( + title=f"Metrics Summary (Last {data['period_days']} days)", show_header=True + ) + summary_table.add_column("Metric", style="cyan") + summary_table.add_column("Value", style="white") + summary_table.add_column("Target", style="yellow") + + # Add key metrics + ttu_metrics = data["ttu_metrics"] + activation_metrics = data["activation_metrics"] + tour_metrics = data["tour_completion_metrics"] + pr_metrics = data["pr_coverage_metrics"] + + summary_table.add_row( + "Average TTU (minutes)", + ( + f"{ttu_metrics['average_ttu_minutes']:.1f}" + if ttu_metrics["average_ttu_minutes"] + else "N/A" + ), + "≤10", + ) + + summary_table.add_row( + "Activation Rate (%)", f"{activation_metrics['activation_rate_percentage']:.1f}", "≥80" + ) + + summary_table.add_row( + "Tour Completion (%)", f"{tour_metrics['completion_rate_percentage']:.1f}", "≥80" + ) + + summary_table.add_row("PR Coverage (%)", f"{pr_metrics['coverage_rate_percentage']:.1f}", "≥90") + + console.print(summary_table) + console.print() + + # Recommendations + recommendations = [] + + if not ttu_metrics["average_ttu_minutes"] or ttu_metrics["average_ttu_minutes"] > 10: + recommendations.append( + "🎯 Focus on reducing TTU: improve onboarding and tutorial experience" + ) + + if activation_metrics["activation_rate_percentage"] < 80: + recommendations.append( + "🚀 Improve activation: make map generation faster and more intuitive" + ) + + if tour_metrics["completion_rate_percentage"] < 80: + recommendations.append("📚 Enhance tour experience: make tours more engaging and shorter") + + if pr_metrics["coverage_rate_percentage"] < 90: + recommendations.append( + "🔀 Increase PR coverage: automate understanding artifact generation" + ) + + if recommendations: + console.print( + Panel("\n".join(recommendations), title="Recommendations", border_style="yellow") + ) diff --git a/cli/ucli/commands/pack_ops.py b/cli/ucli/commands/pack_ops.py new file mode 100644 index 0000000..b19e1c2 --- /dev/null +++ b/cli/ucli/commands/pack_ops.py @@ -0,0 +1,23 @@ +"""Pack create/publish command implementations (extracted from main.py).""" + +from __future__ import annotations + +import os +import pathlib + +from rich import print + +from ucli.pack.publish import make_pack +from ucli.packs.pack import create_pack + + +def run_pack_create(lens: str, tour: str, contracts: str, o: str) -> None: + os.makedirs(pathlib.Path(o).parent, exist_ok=True) + create_pack(lens, tour, contracts, o) + print(f"[green]Wrote[/green] {o}") + + +def run_pack_publish(dist: str = "dist") -> None: + """Build local understanding-pack.zip artifacts.""" + make_pack(dist) + print(f"[green]Pack ready in {dist}/[/green]") diff --git a/cli/ucli/commands/report_ops.py b/cli/ucli/commands/report_ops.py new file mode 100644 index 0000000..ea17903 --- /dev/null +++ b/cli/ucli/commands/report_ops.py @@ -0,0 +1,100 @@ +"""Map / report / glossary / dashboard / ttu / config helpers (extracted from main).""" + +from __future__ import annotations + +import json +import os +import pathlib + +import typer +from rich import print +from rich.console import Console + +from ucli.config import validate_config_dict +from ucli.dashboard.build import build_dashboard +from ucli.glossary.build import build_glossary +from ucli.graph.graph import write_dot +from ucli.metrics.ttu import record as ttu_record +from ucli.metrics.ttu import weekly_report as ttu_weekly +from ucli.report.report import make_report_md + + +def run_map(json_path: str, o: str) -> None: + """Generate DOT graph visualization from analysis results.""" + console = Console() + try: + if not pathlib.Path(json_path).exists(): + console.print(f"[red]Error:[/red] File '{json_path}' does not exist") + raise typer.Exit(1) from None + + with open(json_path, encoding="utf-8") as f: + data = json.load(f) + + os.makedirs(o, exist_ok=True) + dot_path = os.path.join(o, pathlib.Path(json_path).stem + ".dot") + write_dot(data, dot_path) + console.print(f"[green]Wrote[/green] {dot_path}") + except typer.Exit: + raise + except Exception as e: + console.print(f"[red]Error generating map:[/red] {str(e)}") + raise typer.Exit(1) from None + + +def run_report(json_path: str, o: str) -> None: + with open(json_path, encoding="utf-8") as f: + data = json.load(f) + os.makedirs(o, exist_ok=True) + md_path = os.path.join(o, pathlib.Path(json_path).stem + ".md") + with open(md_path, "w", encoding="utf-8") as f: + f.write(make_report_md(data)) + print(f"[green]Wrote[/green] {md_path}") + + +def run_glossary(o: str) -> None: + parent = os.path.dirname(o) + if parent: + os.makedirs(parent, exist_ok=True) + md = build_glossary(".") + with open(o, "w", encoding="utf-8") as f: + f.write(md) + print(f"[green]Wrote[/green] {o}") + + +def run_dashboard(repo: str, lens: str, bounds: str, o: str) -> None: + parent = os.path.dirname(o) + if parent: + os.makedirs(parent, exist_ok=True) + md = build_dashboard({"repo": repo, "lens": lens, "bounds": bounds}) + with open(o, "w", encoding="utf-8") as f: + f.write(md) + print(f"[green]Wrote[/green] {o}") + + +def run_ttu(event: str, o: str) -> None: + if event == "report": + ttu_weekly(o) + print(f"[green]Wrote[/green] {o}") + else: + ttu_record(event) + print(f"[green]Recorded[/green] {event}") + + +def run_config_validate(path: str) -> None: + if not os.path.exists(path): + print(f"[yellow]No config found at {path}[/yellow]") + raise typer.Exit(code=1) from None + import yaml as _yaml + + try: + data = _yaml.safe_load(open(path, encoding="utf-8")) or {} + except Exception as e: + print(f"[red]YAML error:[/red] {e}") + raise typer.Exit(code=1) from None + errors = validate_config_dict(data) + if errors: + print("[red]Config errors:[/red]") + for e in errors: + print(f"- {e}") + raise typer.Exit(code=1) from None + print("[green]Config OK[/green]") diff --git a/cli/ucli/commands/scan_ops.py b/cli/ucli/commands/scan_ops.py new file mode 100644 index 0000000..d72c73d --- /dev/null +++ b/cli/ucli/commands/scan_ops.py @@ -0,0 +1,353 @@ +"""Scan command implementation (extracted from main.py).""" + +from __future__ import annotations + +import json +import pathlib +from typing import Any + +import typer +from rich.console import Console +from rich.panel import Panel +from rich.progress import ( + BarColumn, + Progress, + SpinnerColumn, + TextColumn, + TimeElapsedColumn, +) +from rich.prompt import Confirm, IntPrompt, Prompt +from rich.table import Table + +from ucli.analyzers.registry import ( + build_repo_map, + format_unsupported_summary, + supported_extensions, + supported_languages, +) +from ucli.metrics.ttu import record as ttu_record + + +def _complexity_value(raw: Any) -> int: + if isinstance(raw, dict): + return int(raw.get("cyclomatic", 0) or 0) + try: + return int(raw or 0) + except (TypeError, ValueError): + return 0 + + +def _complexity_kind_label(result: dict[str, Any]) -> str: + """Honest human label for scan UI (keyword vs structural).""" + kind = str(result.get("complexity_kind") or "") + analyzer = str(result.get("analyzer") or "") + if kind == "mccabe": + return "McCabe (Python AST)" + if kind == "ast-cyclomatic" or ( + analyzer.endswith("-ast") and analyzer != "python-ast" + ): + return "ast-cyclomatic (JS/TS/Go structural decision points — not Python McCabe)" + if kind == "keyword-heuristic" or "best-effort" in analyzer: + return "keyword-heuristic (JS/TS/Go/Java regex — not structural AST)" + if kind == "mixed": + return "mixed (do not compare across languages as one scale)" + if kind == "none": + return "n/a" + return kind or "unknown" + + +def run_scan( + path: str, + o: str, + *, + verbose: bool = False, + debug: bool = False, + interactive: bool = False, + lang: str | None = None, +) -> None: + """Scan codebase and build repository map with enhanced progress tracking.""" + console = Console() + + try: + p = pathlib.Path(path) + lang_filter = [x.strip() for x in lang.split(",")] if lang else None + + # Validate input path + if not p.exists(): + console.print(f"[red]❌ Error:[/red] Path '{path}' does not exist") + console.print( + "[yellow]💡 Tip:[/yellow] Use [cyan]u init[/cyan] to set up a new project" + ) + raise typer.Exit(1) from None + + exts = supported_extensions() + has_supported = False + if p.is_file(): + has_supported = p.suffix.lower() in exts + else: + for ext in exts: + if any(p.rglob(f"*{ext}")): + has_supported = True + break + if not has_supported: + console.print( + f"[yellow]⚠️ Warning:[/yellow] No supported source files found in '{path}'" + ) + console.print( + f"[dim]Supported extensions: {', '.join(sorted(exts))} " + f"(languages: {', '.join(supported_languages())})[/dim]" + ) + if not Confirm.ask("Continue anyway?"): + raise typer.Exit(0) from None + + # Interactive mode enhancements + if interactive: + console.print("\n[bold cyan]🔧 Interactive Scan Configuration[/bold cyan]\n") + + # Ask for scan options + Confirm.ask("Include test files?", default=True) + Confirm.ask("Include documentation files?", default=True) + IntPrompt.ask("Maximum analysis depth (1-10)", default=5) + + # Update output path if needed + custom_output = Prompt.ask("Output file path", default=o) + if custom_output != o: + o = custom_output + + # Load project config if available + config_file = p / "understand-first.json" + config = {} + if config_file.exists(): + try: + with open(config_file, encoding="utf-8") as f: + config = json.load(f) + if verbose: + console.print(f"[dim]Loaded configuration from {config_file}[/dim]") + except Exception as e: + console.print(f"[yellow]Warning:[/yellow] Could not load config: {e}") + + # Show scan configuration + config_table = Table(title="📋 Scan Configuration", show_header=False, box=None) + config_table.add_column("Setting", style="cyan", width=20) + config_table.add_column("Value", style="white") + config_table.add_row("Target Path", str(p.absolute())) + config_table.add_row("Output File", o) + config_table.add_row("Verbose Mode", "✅ Enabled" if verbose else "❌ Disabled") + config_table.add_row("Debug Mode", "✅ Enabled" if debug else "❌ Disabled") + config_table.add_row( + "Language filter", + ",".join(lang_filter) if lang_filter else "all registered", + ) + + if config: + config_table.add_row("Project", config.get("project", {}).get("name", "Unknown")) + config_table.add_row( + "Language", config.get("project", {}).get("language", "Auto-detect") + ) + + console.print(config_table) + console.print() + + # Enhanced progress tracking with better error handling + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + BarColumn(), + TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), + TimeElapsedColumn(), + console=console, + ) as progress: + # Task 1: Discover files + discover_task = progress.add_task("🔍 Discovering source files...", total=100) + + try: + from ucli.analyzers.registry import ( + discover_source_inventory, + normalize_lang_filter, + ) + + by_lang, unsupported = discover_source_inventory(p) + if lang_filter: + wanted = normalize_lang_filter(lang_filter) or set() + total_files = sum( + len(files) for lang_id, files in by_lang.items() if lang_id in wanted + ) + else: + total_files = sum(len(files) for files in by_lang.values()) + + if total_files == 0: + console.print( + "[red]❌ No analyzable source files found for the current filter.[/red]\n" + f"[dim]Registered languages: {', '.join(supported_languages())}. " + "Unsupported extensions are reported, not faked.[/dim]" + ) + if unsupported: + console.print(f"[dim]{format_unsupported_summary(unsupported)}[/dim]") + raise typer.Exit(1) from None + + progress.update(discover_task, advance=100) + + if verbose: + for lang_id, files in by_lang.items(): + if files: + console.print(f"[dim]{lang_id}: {len(files)} file(s)[/dim]") + if unsupported: + console.print(f"[dim]{format_unsupported_summary(unsupported)}[/dim]") + + except typer.Exit: + raise + except Exception as e: + progress.update(discover_task, advance=100) + if debug: + console.print(f"[red]File discovery error:[/red] {e}") + else: + console.print( + "[yellow]⚠️ File discovery encountered issues, continuing...[/yellow]" + ) + + # Task 2: Parse and analyze + parse_task = progress.add_task("🧠 Analyzing code structure...", total=100) + + try: + if verbose: + console.print("[dim]Building repository map via language adapters...[/dim]") + + result = build_repo_map(p, languages=lang_filter) + + if not result or not result.get("functions"): + console.print("[yellow]⚠️ Analysis completed but no functions found[/yellow]") + console.print( + "[dim]Check path, --lang filter, and that files match a registered adapter[/dim]" + ) + + progress.update(parse_task, advance=100) + + except Exception as e: + progress.update(parse_task, advance=100) + console.print(f"[red]❌ Analysis failed:[/red] {str(e)}") + if debug: + import traceback + + console.print(f"[dim]{traceback.format_exc()}[/dim]") + raise typer.Exit(1) from None + + # Task 3: Write output + write_task = progress.add_task("💾 Writing analysis results...", total=100) + + try: + output_path = pathlib.Path(o) + output_path.parent.mkdir(parents=True, exist_ok=True) + + with open(output_path, "w", encoding="utf-8") as f: + json.dump(result, f, indent=2) + + progress.update(write_task, advance=100) + + if verbose: + console.print(f"[dim]Results written to {output_path.absolute()}[/dim]") + + except Exception as e: + progress.update(write_task, advance=100) + console.print(f"[red]❌ Failed to write output:[/red] {str(e)}") + raise typer.Exit(1) from None + + # Show results summary with enhanced information + functions_count = len(result.get("functions", {})) + modules_count = len( + set(func.get("file", "") for func in result.get("functions", {}).values()) + ) + analyzed_langs = result.get("languages_analyzed") or [] + unsupported = result.get("unsupported") or {} + + # Average complexity only over analyzed functions (never invent for unsupported langs) + complexities = [ + _complexity_value(func.get("complexity", 0)) + for func in result.get("functions", {}).values() + ] + avg_complexity = sum(complexities) / len(complexities) if complexities else None + high_complexity_count = sum(1 for c in complexities if c > 7) + + fidelity = result.get("analyzer_fidelity") or {} + fidelity_bits = ", ".join(f"{k}={v}" for k, v in fidelity.items()) or "n/a" + analyzer_id = result.get("analyzer") or "n/a" + complexity_label = _complexity_kind_label(result) + avg_line = ( + f" • Average complexity (analyzed funcs): {avg_complexity:.1f}\n" + if avg_complexity is not None + else " • Average complexity: n/a (no analyzed functions)\n" + ) + unsup_line = "" + if unsupported: + unsup_line = f" • {format_unsupported_summary(unsupported)}\n" + + results_panel = Panel( + f"[green]✅ Scan completed successfully![/green]\n\n" + f"📊 [bold]Analysis Results:[/bold]\n" + f" • Languages analyzed: {', '.join(analyzed_langs) or 'none'}\n" + f" • Analyzer: {analyzer_id}\n" + f" • Analyzer fidelity: {fidelity_bits}\n" + f" • Complexity kind: {complexity_label}\n" + f" • Functions analyzed: {functions_count}\n" + f" • Modules processed: {modules_count}\n" + f"{avg_line}" + f" • High complexity functions: {high_complexity_count}\n" + f"{unsup_line}" + f" • Output file: [cyan]{o}[/cyan]\n\n" + f"🎯 [bold]Recommended Next Steps:[/bold]\n" + f" • [cyan]u lens from-seeds --map {o} --seed [/cyan] - Create focused analysis\n" + f" • [cyan]u tour {o.replace('.json', '_lens.json')}[/cyan] - Generate understanding tour\n" + f" • [cyan]u report {o}[/cyan] - Generate detailed report\n" + f" • [cyan]u map {o}[/cyan] - Create visual graph", + title="🎉 Scan Results", + border_style="green", + ) + + console.print(results_panel) + + # Show warnings if any + if high_complexity_count > 0: + console.print( + f"\n[yellow]⚠️ Found {high_complexity_count} high-complexity functions (>7)[/yellow]" + ) + console.print("[dim]Consider reviewing these for refactoring opportunities[/dim]") + + # Track TTU metric + try: + ttu_record( + "scan_completed", + { + "path": str(p), + "functions_count": functions_count, + "modules_count": modules_count, + "avg_complexity": avg_complexity, + "high_complexity_count": high_complexity_count, + "languages_analyzed": analyzed_langs, + }, + ) + except Exception as e: + if debug: + console.print(f"[dim]Failed to track metrics: {e}[/dim]") + + except KeyboardInterrupt: + console.print("\n[yellow]⚠️ Scan cancelled by user[/yellow]") + raise typer.Exit(0) from None + except typer.Exit: + raise + except Exception as e: + console.print(f"[red]❌ Error during scan:[/red] {str(e)}") + + # Provide helpful error messages + if "Permission denied" in str(e): + console.print( + "[yellow]💡 Tip:[/yellow] Check file permissions or try running with different privileges" + ) + elif "No such file" in str(e): + console.print("[yellow]💡 Tip:[/yellow] Verify the path exists and is accessible") + elif "JSON" in str(e): + console.print("[yellow]💡 Tip:[/yellow] Check if the output directory is writable") + + if debug: + import traceback + + console.print(f"[dim]{traceback.format_exc()}[/dim]") + raise typer.Exit(1) from None diff --git a/cli/ucli/commands/tour_ops.py b/cli/ucli/commands/tour_ops.py new file mode 100644 index 0000000..7018c11 --- /dev/null +++ b/cli/ucli/commands/tour_ops.py @@ -0,0 +1,108 @@ +"""Tour / tour_run / tour_gate command implementations (extracted from main.py).""" + +from __future__ import annotations + +import json +import os +import pathlib +import subprocess # nosec B404 # intentional: tour_run fixed argv [sys.executable, fixture] +import sys +from typing import Any + +import typer +from rich import print + + +def write_tour_progress( + progress_json: str = ".uf-progress.json", + *, + opened: int | None = None, + ran: int | None = None, + source: str = "tour_run", +) -> None: + data: dict[str, Any] = {"opened": 0, "ran": 0, "source": source} + if os.path.exists(progress_json): + try: + with open(progress_json, encoding="utf-8") as pf: + data.update(json.load(pf)) + except (OSError, json.JSONDecodeError, TypeError, ValueError): + pass + if opened is not None: + data["opened"] = max(int(data.get("opened", 0)), opened) + if ran is not None: + data["ran"] = max(int(data.get("ran", 0)), ran) + data["source"] = source + with open(progress_json, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2) + + +def run_tour(lens_json: str, o: str) -> None: + from ucli.lens.lens import write_tour_md + + with open(lens_json, encoding="utf-8") as f: + lens = json.load(f) + os.makedirs(pathlib.Path(o).parent, exist_ok=True) + md = write_tour_md(lens) + with open(o, "w", encoding="utf-8") as f: + f.write(md) + print(f"[green]Wrote[/green] {o}") + + +def run_tour_run( + lens_json: str, + fixtures_dir: str = "fixtures", +) -> None: + """Attempt to run a minimal fixture and verify runtime hits align with the lens.""" + from ucli.analyzers.map_meta import lens_runtime_trace_supported + + with open(lens_json, encoding="utf-8") as f: + lens = json.load(f) + if not lens_runtime_trace_supported(lens): + lang = lens.get("language") or lens.get("lens", {}).get("language") or "non-python" + print( + "[red]tour_run refused:[/red] runtime fixtures / `u trace` are Python-only.\n" + f"[dim]This lens language={lang!r} has no `.py` files (or is empty). " + "Use `u tour` for a reading plan from the map; do not invent runtime parity " + "for JS/TS/Go/Java/Rust/C#.[/dim]" + ) + raise typer.Exit(1) from None + fixture = os.path.join(fixtures_dir, "fixture_hot_path.py") + if not os.path.exists(fixture): + print("[yellow]No fixture found; cannot run tour.[/yellow]") + raise typer.Exit(1) from None + rc = subprocess.call([sys.executable, fixture]) # nosec B603 + if rc != 0: + print("[red]Fixture failed.[/red]") + raise typer.Exit(rc) from None + write_tour_progress(opened=3, ran=3, source="tour_run") + print("[green]Fixture ran successfully.[/green]") + + +def run_tour_gate( + progress_json: str = ".uf-progress.json", + fixture_lens: str = "", + fixtures_dir: str = "fixtures", +) -> None: + """Fail if walkthrough milestones are not met (opened>=3 and ran>=3).""" + if fixture_lens: + run_tour_run(fixture_lens, fixtures_dir=fixtures_dir) + try: + data = json.load(open(progress_json, encoding="utf-8")) + except Exception: + print( + "[red]No progress file found[/red]\n" + f"Expected {progress_json}. Open the IDE walkthrough, or run:\n" + " u tour_gate --fixture-lens maps/lens_merged.json\n" + "CI fixture gate: u tour_run maps/lens_merged.json -f fixtures" + ) + raise typer.Exit(1) from None + opened = int(data.get("opened", 0)) + ran = int(data.get("ran", 0)) + if opened >= 3 and ran >= 3: + print("[green]Tour milestones met[/green]") + return + if data.get("source") == "tour_run" and ran >= 1: + print("[green]Tour fixture progress met (tour_run)[/green]") + return + print(f"[red]Tour milestones not met[/red]: opened {opened}/3 ran {ran}/3") + raise typer.Exit(1) from None diff --git a/cli/ucli/commands/trace_ops.py b/cli/ucli/commands/trace_ops.py new file mode 100644 index 0000000..27779ab --- /dev/null +++ b/cli/ucli/commands/trace_ops.py @@ -0,0 +1,38 @@ +"""Trace command implementations (extracted from main.py).""" + +from __future__ import annotations + +import json +import os +import pathlib + +from rich import print + +from ucli.trace.pytrace import analyze_errors_static, run_callable_with_trace + + +def run_trace_module( + pyfile: str, + func: str, + a: str | None = None, + b: str | None = None, + o: str = "traces/trace.json", +) -> None: + os.makedirs(pathlib.Path(o).parent, exist_ok=True) + data = run_callable_with_trace(pyfile, func, a, b) + with open(o, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2) + print(f"[green]Wrote[/green] {o}") + + +def run_trace_errors(pyfile: str, *, json_out: bool = True) -> None: + data = analyze_errors_static(pyfile) + if json_out: + print(json.dumps(data, indent=2)) + else: + print("Raises:") + for r in data.get("raises", []): + print(f" line {r.get('line')}: {r.get('exc')}") + print("Catches:") + for c in data.get("catches", []): + print(f" line {c.get('line')}: {c.get('catch')}") diff --git a/cli/ucli/commands/typer_groups.py b/cli/ucli/commands/typer_groups.py new file mode 100644 index 0000000..0fad28a --- /dev/null +++ b/cli/ucli/commands/typer_groups.py @@ -0,0 +1,451 @@ +"""Typer sub-app wiring for command groups (keeps main.py thin). + +Each ``register_*`` attaches a named Typer group and maps options to ``run_*`` +helpers in the corresponding ``*_ops`` modules. +""" + +from __future__ import annotations + +import typer + +from ucli.commands.ci_ops import run_ci +from ucli.commands.contracts_ops import ( + run_contracts_check, + run_contracts_compose, + run_contracts_from_openapi, + run_contracts_from_proto, + run_contracts_init, + run_contracts_lean_stubs, + run_contracts_report, + run_contracts_stub, + run_contracts_verify_lean, +) +from ucli.commands.demo_ops import run_demo +from ucli.commands.diff_ops import run_diff +from ucli.commands.doctor_ops import run_doctor +from ucli.commands.explorer_ops import run_tui +from ucli.commands.init_ops import run_init +from ucli.commands.lens_ops import ( + run_ingest_github, + run_ingest_jira, + run_lens_explain, + run_lens_from_issue, + run_lens_from_seeds, + run_lens_merge_trace, + run_lens_preset, +) +from ucli.commands.metrics_ops import run_metrics +from ucli.commands.pack_ops import run_pack_create, run_pack_publish +from ucli.commands.report_ops import ( + run_config_validate, + run_dashboard, + run_glossary, + run_map, + run_report, + run_ttu, +) +from ucli.commands.scan_ops import run_scan +from ucli.commands.tour_ops import run_tour, run_tour_gate, run_tour_run +from ucli.commands.trace_ops import run_trace_errors, run_trace_module +from ucli.commands.visual_ops import run_boundaries_scan, run_visual_delta +from ucli.commands.wizard_ops import run_wizard + + +def register_lens(app: typer.Typer) -> None: + lens_app = typer.Typer(help="Task lenses") + app.add_typer(lens_app, name="lens") + + @lens_app.command("from-issue") + def lens_from_issue_cmd( + issue_md: str, + map: str = typer.Option(..., "--map"), + o: str = typer.Option("maps/lens.json", "--output", "-o"), + ): + run_lens_from_issue(issue_md, map, o) + + @lens_app.command("from-seeds") + def lens_from_seeds_cmd( + seed: list[str] = typer.Option([], "--seed"), + map: str = typer.Option(..., "--map"), + o: str = typer.Option("maps/lens.json", "--output", "-o"), + interactive: bool = typer.Option( + False, "--interactive", "-i", help="Interactive seed selection" + ), + verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable verbose output"), + exact: bool = typer.Option( + False, + "--exact", + help="Match seeds exactly (full qname, local name, or simple_name) instead of substring", + ), + ): + """Create understanding lens from seed functions with enhanced TUI.""" + run_lens_from_seeds(seed, map, o, interactive=interactive, verbose=verbose, exact=exact) + + @lens_app.command("merge-trace") + def lens_merge_trace_cmd( + lens_json: str, + trace_json: str, + o: str = typer.Option("maps/lens_merged.json", "--output", "-o"), + ): + run_lens_merge_trace(lens_json, trace_json, o) + + @lens_app.command("preset") + def lens_preset_cmd( + label: str, + map: str = typer.Option(..., "--map"), + o: str = typer.Option("maps/lens.json", "--output", "-o"), + ): + run_lens_preset(label, map, o) + + @lens_app.command("ingest-github") + def ingest_github(log_path: str): + run_ingest_github(log_path) + + @lens_app.command("ingest-jira") + def ingest_jira(jira_path: str): + run_ingest_jira(jira_path) + + @lens_app.command("explain") + def lens_explain( + qname: str, + lens: str = typer.Option("maps/lens_merged.json", "--lens"), + repo: str = typer.Option("maps/repo.json", "--repo"), + json_out: bool = typer.Option(False, "--json"), + ): + run_lens_explain(qname, lens, repo, json_out=json_out) + + +def register_trace(app: typer.Typer) -> None: + trace_app = typer.Typer(help="Runtime tracing (Python demo)") + app.add_typer(trace_app, name="trace") + + @trace_app.command("module") + def trace_module( + pyfile: str, + func: str, + a: str | None = None, + b: str | None = None, + o: str = typer.Option("traces/trace.json", "--output", "-o"), + ): + run_trace_module(pyfile, func, a, b, o) + + @trace_app.command("errors") + def trace_errors(pyfile: str, json_out: bool = typer.Option(True, "--json/--no-json")): + run_trace_errors(pyfile, json_out=json_out) + + +def register_boundaries(app: typer.Typer) -> None: + bound_app = typer.Typer(help="Boundary scanners") + app.add_typer(bound_app, name="boundaries") + + @bound_app.command("scan") + def boundaries_scan( + path: str = typer.Argument("."), + o: str = typer.Option("maps/boundaries.json", "--output", "-o"), + ): + run_boundaries_scan(path, o) + + +def register_contracts(app: typer.Typer) -> None: + contracts_app = typer.Typer(help="Contracts") + app.add_typer(contracts_app, name="contracts") + + @contracts_app.command("init") + def contracts_init( + path: str = typer.Argument("."), + o: str = typer.Option("contracts/contracts.yaml", "--output", "-o"), + ): + run_contracts_init(path, o) + + @contracts_app.command("check") + def contracts_check(path: str): + run_contracts_check(path) + + @contracts_app.command("stub-tests") + def contracts_stub( + path: str, o: str = typer.Option("tests/test_contracts.py", "--output", "-o") + ): + run_contracts_stub(path, o) + + @contracts_app.command("from-openapi") + def contracts_from_openapi( + path: str, + o: str = typer.Option("contracts/contracts_from_openapi.yaml", "--output", "-o"), + ): + run_contracts_from_openapi(path, o) + + @contracts_app.command("from-proto") + def contracts_from_proto( + path: str, + o: str = typer.Option("contracts/contracts_from_proto.yaml", "--output", "-o"), + ): + run_contracts_from_proto(path, o) + + @contracts_app.command( + "lean-stubs", + help="Emit Lean scaffold stubs (Prop := True / by trivial; not a real proof).", + ) + def contracts_lean_stubs( + contracts_yaml: str, o: str = typer.Option("contracts/lean/", "--output-dir", "-o") + ): + run_contracts_lean_stubs(contracts_yaml, o) + + @contracts_app.command("compose") + def contracts_compose( + i: list[str] = typer.Option([], "--input", "-i", help="Contract YAML input paths"), + o: str = typer.Option("contracts/contracts.yaml", "--output", "-o"), + ): + run_contracts_compose(i, o) + + @contracts_app.command( + "verify-lean", + help="Presence-only check that Lean scaffold theorem stubs exist (does not compile Lean).", + ) + def contracts_verify_lean( + contracts_yaml: str = typer.Argument(...), + lean_dir: str = typer.Option("contracts/lean", "--lean-dir", "-l"), + json_out: bool = typer.Option(False, "--json"), + ): + run_contracts_verify_lean(contracts_yaml, lean_dir, json_out=json_out) + + @contracts_app.command("report") + def contracts_report( + path: str, + json_out: bool = typer.Option(False, "--json", help="Emit JSON report to stdout"), + ): + run_contracts_report(path, json_out=json_out) + + +def register_pack(app: typer.Typer) -> None: + pack_app = typer.Typer(help="Understanding Packs") + app.add_typer(pack_app, name="pack") + + @pack_app.command("create") + def pack_create( + lens: str = typer.Option(..., "--lens"), + tour: str = typer.Option(..., "--tour"), + contracts: str = typer.Option(..., "--contracts"), + o: str = typer.Option("packs/pack.zip", "--output", "-o"), + ): + run_pack_create(lens, tour, contracts, o) + + @pack_app.command("publish") + def pack_publish( + dist: str = typer.Option("dist", "--dist", help="Output directory for pack artifacts"), + ): + """Build local understanding-pack.zip artifacts (replaces legacy `u pack --publish`).""" + run_pack_publish(dist) + + +def register_visual(app: typer.Typer) -> None: + visual_app = typer.Typer(help="Visualization utilities") + app.add_typer(visual_app, name="visual") + + @visual_app.command("delta") + def visual_delta( + old_lens: str, + new_lens: str, + o: str = typer.Option("maps/delta.svg", "--output", "-o"), + ): + run_visual_delta(old_lens, new_lens, o) + + +def register_top_level(app: typer.Typer) -> None: + """Register flat top-level commands (same pattern as named groups).""" + + @app.command() + def scan( + path: str = typer.Argument("."), + o: str = typer.Option("maps/out.json", "--output", "-o"), + verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable verbose output"), + debug: bool = typer.Option(False, "--debug", help="Enable debug mode with tracebacks"), + interactive: bool = typer.Option( + False, "--interactive", "-i", help="Interactive scan with guided options" + ), + lang: str | None = typer.Option( + None, + "--lang", + help=( + "Limit analyzers: python, javascript, go, java, rust, csharp " + "(aliases: js, ts, typescript, golang, cs, c#). Comma-separated." + ), + ), + ): + """Scan codebase and build repository map with enhanced progress tracking.""" + run_scan(path, o, verbose=verbose, debug=debug, interactive=interactive, lang=lang) + + @app.command() + def map(json_path: str, o: str = typer.Option("maps", "--output", "-o")): + """Generate DOT graph visualization from analysis results.""" + run_map(json_path, o) + + @app.command() + def report(json_path: str, o: str = typer.Option("maps", "--output", "-o")): + run_report(json_path, o) + + @app.command() + def tour(lens_json: str, o: str = typer.Option("tours/tour.md", "--output", "-o")): + run_tour(lens_json, o) + + # Keep underscore names so docs/CI (`u tour_run`) match invocation (Typer + # would otherwise expose only ``tour-run``). + @app.command("tour_run") + def tour_run(lens_json: str, fixtures_dir: str = typer.Option("fixtures", "--fixtures", "-f")): + """Run a Python fixture against a lens; refuses non-Python maps (no invented runtime).""" + run_tour_run(lens_json, fixtures_dir=fixtures_dir) + + @app.command() + def glossary(o: str = typer.Option("docs/glossary.md", "--output", "-o")): + run_glossary(o) + + @app.command() + def dashboard( + repo: str = typer.Option("maps/repo.json", "--repo"), + lens: str = typer.Option("maps/lens_merged.json", "--lens"), + bounds: str = typer.Option("maps/boundaries.json", "--bounds"), + o: str = typer.Option("docs/understanding-dashboard.md", "--output", "-o"), + ): + run_dashboard(repo, lens, bounds, o) + + @app.command() + def ttu( + event: str = typer.Argument(...), + o: str = typer.Option("docs/ttu.md", "--output", "-o"), + ): + run_ttu(event, o) + + @app.command() + def lens_preset( + label: str, + map: str = typer.Option(..., "--map"), + o: str = typer.Option("maps/lens.json", "--output", "-o"), + ): + run_lens_preset(label, map, o) + + @app.command() + def doctor(): + run_doctor() + + @app.command() + def demo(): + """Generate sample maps, lens, tour, and dashboard artifacts.""" + run_demo() + + @app.command() + def init( + stack: str = typer.Option("py", "--stack"), + ci: str = typer.Option("github", "--ci"), + wizard: bool = typer.Option(False, "--wizard", help="Interactive configuration wizard"), + tui: bool = typer.Option(False, "--tui", help="Launch interactive TUI mode"), + ): + """Write `.understand-first.yml` (use --wizard for interactive setup).""" + run_init(stack=stack, ci=ci, wizard=wizard, tui=tui) + + @app.command("tour_gate") + def tour_gate( + progress_json: str = typer.Option(".uf-progress.json", "--progress"), + fixture_lens: str = typer.Option( + "", + "--fixture-lens", + help="If set, run tour_run on this lens and write progress (CI-friendly).", + ), + fixtures_dir: str = typer.Option("fixtures", "--fixtures", "-f"), + ): + """Fail if walkthrough milestones are not met (opened>=3 and ran>=3). + + Progress is produced by: + - VS Code walkthrough panel (writes .uf-progress.json), or + - ``u tour_gate --fixture-lens maps/lens_merged.json`` (runs fixture, records ran), or + - ``u tour_run`` (updates ran count after a successful fixture). + + For PR CI that only needs a fixture smoke test, prefer ``u tour_run`` directly + (see tour-must-pass.yml). This gate is for milestone tracking when a progress + file exists. + """ + run_tour_gate(progress_json, fixture_lens=fixture_lens, fixtures_dir=fixtures_dir) + + @app.command() + def diff( + old_lens: str = typer.Option(..., "--old"), + new_lens: str = typer.Option(..., "--new"), + o: str = typer.Option("maps/delta.svg", "--output", "-o"), + verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable verbose output"), + json_output: bool = typer.Option(False, "--json", help="Output JSON delta instead of SVG"), + markdown: bool = typer.Option(False, "--markdown", help="Output Markdown summary"), + ci_gate: bool = typer.Option( + False, "--ci-gate", help="Exit with non-zero code for CI gates" + ), + policy_threshold: int = typer.Option( + 5, "--policy-threshold", help="Complexity threshold for policy breach" + ), + ): + """Compare two lens files and generate delta visualization with enhanced analysis.""" + run_diff( + old_lens, + new_lens, + o, + verbose=verbose, + json_output=json_output, + markdown=markdown, + ci_gate=ci_gate, + policy_threshold=policy_threshold, + ) + + @app.command() + def metrics( + dashboard: bool = typer.Option(False, "--dashboard", "-d", help="Show metrics dashboard"), + days: int = typer.Option(30, "--days", help="Number of days to analyze"), + export: str = typer.Option(None, "--export", "-e", help="Export metrics to file"), + track: str = typer.Option(None, "--track", "-t", help="Track a specific event"), + ): + """View and manage Understand-First metrics for TTU/TTFSC goals.""" + run_metrics(dashboard=dashboard, days=days, export=export, track=track) + + @app.command() + def config_validate(path: str = typer.Option(".understand-first.yml", "--path")): + run_config_validate(path) + + @app.command() + def ci( + scan_path: str = typer.Option(".", "--scan", help="Path to scan for analysis"), + output_dir: str = typer.Option( + "ci-artifacts", "--output", "-o", help="Output directory for CI artifacts" + ), + fail_on_issues: bool = typer.Option( + True, "--fail-on-issues", help="Fail CI on understanding issues" + ), + generate_report: bool = typer.Option(True, "--report", help="Generate CI report"), + ): + """Run Understand-First analysis for CI/CD pipeline with enhanced reporting.""" + run_ci( + scan_path, + output_dir, + fail_on_issues=fail_on_issues, + generate_report=generate_report, + ) + + @app.command() + def wizard( + scan_path: str = typer.Option(".", "--scan", help="Path to scan for analysis"), + interactive: bool = typer.Option(True, "--interactive", help="Run in interactive mode"), + ): + """Interactive wizard to guide users through understanding analysis.""" + run_wizard(scan_path, interactive=interactive) + + @app.command() + def tui( + scan_path: str = typer.Option(".", "--scan", help="Path to scan for analysis"), + ): + """Launch interactive Text User Interface for code understanding.""" + run_tui(scan_path) + + +def register_all_groups(app: typer.Typer) -> None: + """Attach lens/trace/boundaries/contracts/pack/visual groups and top-level cmds.""" + register_lens(app) + register_trace(app) + register_boundaries(app) + register_contracts(app) + register_pack(app) + register_visual(app) + register_top_level(app) diff --git a/cli/ucli/commands/visual_ops.py b/cli/ucli/commands/visual_ops.py new file mode 100644 index 0000000..f32123a --- /dev/null +++ b/cli/ucli/commands/visual_ops.py @@ -0,0 +1,28 @@ +"""Visual delta and boundaries scan (extracted from main.py).""" + +from __future__ import annotations + +import json +import os +import pathlib + +from rich import print + +from ucli.boundaries.scan import scan_boundaries +from ucli.visual.delta import lens_delta_svg + + +def run_visual_delta(old_lens: str, new_lens: str, o: str) -> None: + svg = lens_delta_svg(old_lens, new_lens) + os.makedirs(pathlib.Path(o).parent, exist_ok=True) + with open(o, "w", encoding="utf-8") as f: + f.write(svg) + print(f"[green]Wrote[/green] {o}") + + +def run_boundaries_scan(path: str, o: str) -> None: + result = scan_boundaries(path) + os.makedirs(pathlib.Path(o).parent, exist_ok=True) + with open(o, "w", encoding="utf-8") as f: + json.dump(result, f, indent=2) + print(f"[green]Wrote[/green] {o}") diff --git a/cli/ucli/commands/wizard_ops.py b/cli/ucli/commands/wizard_ops.py new file mode 100644 index 0000000..256fb90 --- /dev/null +++ b/cli/ucli/commands/wizard_ops.py @@ -0,0 +1,354 @@ +"""Interactive wizard command (extracted from main.py).""" + +from __future__ import annotations + +import pathlib +import time + +import typer +from rich.console import Console +from rich.panel import Panel +from rich.progress import ( + BarColumn, + Progress, + SpinnerColumn, + TextColumn, + TimeElapsedColumn, +) +from rich.table import Table + +from ucli.analyzers.registry import build_repo_map +from ucli.lens.lens import lens_from_seeds, rank_by_error_proximity, write_tour_md +from ucli.metrics.ttu import record as ttu_record + + +def _complexity_value(raw) -> int: + if isinstance(raw, dict): + return int(raw.get("cyclomatic", 0) or 0) + try: + return int(raw or 0) + except (TypeError, ValueError): + return 0 + + +def run_wizard(scan_path: str, *, interactive: bool = True) -> None: + """Interactive wizard to guide users through understanding analysis.""" + console = Console() + + if not interactive: + console.print("[yellow]Non-interactive mode - running basic analysis[/yellow]") + result = build_repo_map(pathlib.Path(scan_path)) + console.print( + f"[green]Analysis complete. Found {len(result.get('functions', {}))} functions.[/green]" + ) + return + + console.print( + Panel.fit( + "[bold blue]🧙 Welcome to the Understand-First Wizard![/bold blue]\n\n" + "This wizard will guide you through understanding your codebase step by step.\n" + "We'll help you identify key functions, understand their relationships,\n" + "and create a personalized learning path.", + title="Getting Started", + border_style="blue", + ) + ) + + try: + console.print("\n[bold cyan]Step 1: Scanning your codebase...[/bold cyan]") + + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + BarColumn(), + TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), + TimeElapsedColumn(), + console=console, + ) as progress: + scan_task = progress.add_task("Analyzing code structure...", total=100) + + progress.update(scan_task, advance=20) + time.sleep(0.5) + + progress.update(scan_task, advance=30) + result = build_repo_map(pathlib.Path(scan_path)) + + progress.update(scan_task, advance=30) + time.sleep(0.3) + + progress.update(scan_task, advance=20) + + functions = result.get("functions", {}) + + if functions: + high_complexity = sum( + 1 for f in functions.values() if _complexity_value(f.get("complexity", 0)) > 5 + ) + with_side_effects = sum(1 for f in functions.values() if f.get("side_effects", [])) + + console.print( + Panel.fit( + f"[green]✓ Analysis complete![/green]\n\n" + f"• [bold]{len(functions)}[/bold] functions found\n" + f"• [bold]{high_complexity}[/bold] high complexity functions\n" + f"• [bold]{with_side_effects}[/bold] functions with side effects", + title="Analysis Results", + border_style="green", + ) + ) + else: + console.print( + Panel.fit( + "[yellow]⚠ No functions found in the specified path[/yellow]\n\n" + "This could mean:\n" + "• The path doesn't contain Python files\n" + "• The files don't have function definitions\n" + "• There's an issue with the file permissions", + title="No Functions Found", + border_style="yellow", + ) + ) + return + + console.print("\n[bold cyan]Step 2: Function Overview[/bold cyan]") + + table = Table(title="Functions in your codebase") + table.add_column("Name", style="cyan", no_wrap=True) + table.add_column("Complexity", justify="center", style="magenta") + table.add_column("Side Effects", justify="center", style="yellow") + table.add_column("Lines", justify="center", style="green") + + for func_name, func_data in list(functions.items())[:10]: + complexity = _complexity_value(func_data.get("complexity", 0)) + side_effects = func_data.get("side_effects", []) + lines = func_data.get("lines", 0) + + complexity_str = str(complexity) + if complexity > 5: + complexity_str = f"[red]{complexity}[/red]" + elif complexity > 3: + complexity_str = f"[yellow]{complexity}[/yellow]" + + effects_str = str(len(side_effects)) if side_effects else "0" + if side_effects: + effects_str = f"[yellow]{len(side_effects)}[/yellow]" + + table.add_row(func_name, complexity_str, effects_str, str(lines)) + + console.print(table) + + if len(functions) > 10: + console.print(f"\n[yellow]... and {len(functions) - 10} more functions[/yellow]") + + console.print("\n[bold cyan]Step 3: Select Seed Functions[/bold cyan]") + console.print( + "Choose functions to start your understanding journey. These will be your 'seeds'.\n" + "Seeds help us understand which functions are most important to you.\n" + ) + + seeds: list[str] = [] + while True: + if seeds: + console.print(f"[green]Current seeds: {', '.join(seeds)}[/green]") + else: + console.print("[yellow]No seeds selected yet[/yellow]") + + choice = typer.prompt( + "Enter function name to add as seed (or 'done' to continue, 'list' to see all functions, 'help' for tips)", + default="done", + ) + + if choice.lower() == "done": + break + if choice.lower() == "list": + console.print("\n[bold]All functions:[/bold]") + for i, func_name in enumerate(functions.keys(), 1): + complexity = _complexity_value(functions[func_name].get("complexity", 0)) + console.print(f" {i:2d}. {func_name} (complexity: {complexity})") + console.print() + continue + if choice.lower() == "help": + console.print("\n[bold]Tips for selecting seeds:[/bold]") + console.print("• Choose functions you're most interested in understanding") + console.print("• Pick functions that seem central to your codebase") + console.print("• Consider functions with high complexity or side effects") + console.print("• You can always add more seeds later\n") + continue + if choice in functions: + if choice not in seeds: + seeds.append(choice) + console.print(f"[green]✓ Added '{choice}' as seed[/green]") + else: + console.print(f"[yellow]'{choice}' is already a seed[/yellow]") + else: + console.print( + f"[red]Function '{choice}' not found. Try 'list' to see available functions.[/red]" + ) + + if not seeds: + seeds = list(functions.keys())[:3] + console.print(f"[yellow]No seeds selected. Using first 3 functions: {seeds}[/yellow]") + console.print( + "[dim]You can always run the wizard again to select different seeds[/dim]" + ) + + console.print(f"\n[green]✓ Selected seeds: {seeds}[/green]\n") + + console.print("[bold cyan]Step 4: Generating Understanding Lens...[/bold cyan]") + + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + BarColumn(), + TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), + TimeElapsedColumn(), + console=console, + ) as progress: + lens_task = progress.add_task("Building understanding lens...", total=100) + + progress.update(lens_task, advance=25) + time.sleep(0.3) + + progress.update(lens_task, advance=25) + lens = lens_from_seeds(seeds, result) + + progress.update(lens_task, advance=25) + rank_by_error_proximity(lens) + + progress.update(lens_task, advance=25) + + lens_functions = lens.get("functions", {}) + + console.print( + Panel.fit( + f"[green]✓ Generated understanding lens![/green]\n\n" + f"• [bold]{len(lens_functions)}[/bold] functions in your lens\n" + f"• Functions ranked by importance and error proximity\n" + f"• Ready for tour generation", + title="Lens Generated", + border_style="green", + ) + ) + + console.print("\n[bold cyan]Step 5: Understanding Lens Results[/bold cyan]") + console.print("Functions in your understanding lens (ranked by importance):\n") + + lens_table = Table(title="Understanding Lens Functions") + lens_table.add_column("Rank", justify="center", style="cyan") + lens_table.add_column("Function", style="cyan", no_wrap=True) + lens_table.add_column("Complexity", justify="center", style="magenta") + lens_table.add_column("Side Effects", justify="center", style="yellow") + lens_table.add_column("Importance", justify="center", style="green") + + for i, (func_name, func_data) in enumerate(lens_functions.items(), 1): + complexity = _complexity_value(func_data.get("complexity", 0)) + side_effects = func_data.get("side_effects", []) + importance = func_data.get("importance", 0) + + complexity_str = str(complexity) + if complexity > 5: + complexity_str = f"[red]{complexity}[/red]" + elif complexity > 3: + complexity_str = f"[yellow]{complexity}[/yellow]" + + effects_str = str(len(side_effects)) if side_effects else "0" + if side_effects: + effects_str = f"[yellow]{len(side_effects)}[/yellow]" + + importance_str = f"{importance:.1f}" if importance else "N/A" + + lens_table.add_row(str(i), func_name, complexity_str, effects_str, importance_str) + + console.print(lens_table) + + console.print("\n[bold cyan]Step 6: Generating Understanding Tour...[/bold cyan]") + + if lens_functions: + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + BarColumn(), + TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), + TimeElapsedColumn(), + console=console, + ) as progress: + tour_task = progress.add_task("Generating understanding tour...", total=100) + + progress.update(tour_task, advance=50) + tour_md = write_tour_md(lens) + + progress.update(tour_task, advance=30) + tour_path = "understanding_tour.md" + with open(tour_path, "w", encoding="utf-8") as f: + f.write(tour_md) + + progress.update(tour_task, advance=20) + + console.print( + Panel.fit( + f"[green]✓ Tour generated successfully![/green]\n\n" + f"• File: [bold]{tour_path}[/bold]\n" + f"• Contains step-by-step guidance\n" + f"• Ready for review and learning", + title="Tour Generated", + border_style="green", + ) + ) + else: + console.print( + Panel.fit( + "[yellow]⚠ No functions in lens for tour generation[/yellow]\n\n" + "This usually means the selected seeds didn't have enough\n" + "connections to other functions in your codebase.", + title="No Tour Generated", + border_style="yellow", + ) + ) + + console.print("\n[bold green]🎉 Wizard Complete![/bold green]") + + summary_table = Table(title="What you've accomplished") + summary_table.add_column("Metric", style="cyan") + summary_table.add_column("Value", style="green") + + summary_table.add_row("Functions analyzed", str(len(functions))) + summary_table.add_row("Seeds selected", str(len(seeds))) + summary_table.add_row("Lens functions", str(len(lens_functions))) + summary_table.add_row("Tour generated", "Yes" if lens_functions else "No") + + console.print(summary_table) + + console.print("\n[bold cyan]Next steps:[/bold cyan]") + console.print(" • Review the understanding tour to understand your code") + console.print(" • Use 'u map' to visualize the code structure") + console.print(" • Use 'u diff' to track changes over time") + console.print(" • Use 'u ci' to integrate with your CI/CD pipeline") + console.print(" • Use 'u tui' for an interactive exploration experience") + + ttu_record( + "wizard_completed", + { + "functions_analyzed": len(functions), + "seeds_selected": len(seeds), + "lens_functions": len(lens_functions), + "tour_generated": bool(lens_functions), + }, + ) + + except KeyboardInterrupt: + console.print("\n[yellow]Wizard interrupted by user[/yellow]") + console.print("[dim]You can run 'u wizard' again anytime to continue[/dim]") + except Exception as e: + console.print( + Panel.fit( + f"[red]Error during wizard:[/red] {str(e)}\n\n" + "This might be due to:\n" + "• File permission issues\n" + "• Corrupted code files\n" + "• Missing dependencies\n\n" + "Try running with --interactive=false for basic analysis", + title="Wizard Error", + border_style="red", + ) + ) + raise typer.Exit(1) from None diff --git a/cli/ucli/config.py b/cli/ucli/config.py index 8438537..bd04ac5 100644 --- a/cli/ucli/config.py +++ b/cli/ucli/config.py @@ -1,4 +1,5 @@ -import os, re +import os + import yaml from jsonschema import Draft202012Validator @@ -36,7 +37,7 @@ def load_config(path: str = ".understand-first.yml") -> dict: if not os.path.exists(path): return cfg try: - data = yaml.safe_load(open(path, "r", encoding="utf-8")) or {} + data = yaml.safe_load(open(path, encoding="utf-8")) or {} if isinstance(data, dict): cfg.update( {k: v for k, v in data.items() if k in cfg or k == "seeds_for" or k == "metrics"} @@ -52,10 +53,38 @@ def load_preset(label: str, path: str = ".understand-first.yml") -> list: return (c.get("seeds_for", {}) or {}).get(label, []) +def schema_keys() -> set: + return set(SCHEMA["properties"].keys()) + + +def filter_config_to_schema(data: dict) -> dict: + """Keep only keys honored by runtime SCHEMA (source of truth).""" + if not isinstance(data, dict): + return {"hops": 2} + out = {} + for key in schema_keys(): + if key not in data: + continue + value = data[key] + if key == "metrics": + # SCHEMA only allows metrics.enabled + enabled = False + if isinstance(value, dict): + enabled = bool(value.get("enabled", False)) + elif isinstance(value, bool): + enabled = value + out[key] = {"enabled": enabled} + else: + out[key] = value + if "hops" not in out: + out["hops"] = 2 + return out + + def validate_config_dict(data: dict) -> list: validator = Draft202012Validator(SCHEMA) errors = [] - valid_keys = set(SCHEMA["properties"].keys()) + valid_keys = schema_keys() for err in validator.iter_errors(data): msg = err.message if err.validator == "additionalProperties": diff --git a/cli/ucli/contracts/contracts.py b/cli/ucli/contracts/contracts.py index 30c53df..318f785 100644 --- a/cli/ucli/contracts/contracts.py +++ b/cli/ucli/contracts/contracts.py @@ -1,22 +1,21 @@ +import json import os import re -import importlib.util -import json -from typing import Tuple, Any, Dict, List +import re as _re from collections import OrderedDict +from typing import Any + import yaml -import re as _re try: - from openapi_schema_validator.validators import OAS30Validator - from openapi_schema_validator.validators import check_openapi_schema + from openapi_schema_validator.validators import OAS30Validator, check_openapi_schema except Exception: # pragma: no cover OAS30Validator = None # type: ignore check_openapi_schema = None # type: ignore def _read(path: str) -> str: - return open(path, "r", encoding="utf-8", errors="ignore").read() + return open(path, encoding="utf-8", errors="ignore").read() FN_DEF_RE = _re.compile(r"^def\s+([A-Za-z_][A-Za-z0-9_]*)\(", _re.M) @@ -30,7 +29,7 @@ def _read(path: str) -> str: def init_contracts(root: str) -> str: entries = [] - for r, d, files in os.walk(root): + for r, _d, files in os.walk(root): for f in files: if f.endswith(".py"): full = os.path.join(r, f) @@ -51,18 +50,31 @@ def init_contracts(root: str) -> str: def _exists_module_func(module_path: str, fn: str) -> bool: + """AST presence check only — does not exec user modules (safer than import).""" try: - spec = importlib.util.spec_from_file_location("_mod", module_path) - if spec is None or spec.loader is None: + import ast + from pathlib import Path + + path = Path(module_path) + if not path.is_file(): return False - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - return hasattr(mod, fn) + tree = ast.parse(path.read_text(encoding="utf-8")) + for node in tree.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == fn: + return True + if isinstance(node, ast.ClassDef): + for item in node.body: + if ( + isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)) + and item.name == fn + ): + return True + return False except Exception: return False -def check_contracts(path: str) -> Tuple[bool, str]: +def check_contracts(path: str) -> tuple[bool, str]: txt = _read(path) ok = True report_lines = ["# Contracts Check Report\n"] @@ -84,7 +96,7 @@ def check_contracts(path: str) -> Tuple[bool, str]: return ok, "\n".join(report_lines) -def _strategy_expr(meta: Dict[str, Any]) -> str: +def _strategy_expr(meta: dict[str, Any]) -> str: t = (meta or {}).get("type") if "enum" in (meta or {}): return "st.sampled_from(" + json.dumps(meta["enum"]) + ")" @@ -130,12 +142,25 @@ def _strategy_expr(meta: Dict[str, Any]) -> str: def stub_tests(path: str) -> str: + """Generate Hypothesis-oriented stubs that check symbols via AST (no exec_module).""" txt = _read(path) out = [ "# Auto-generated contract stubs\n", "# Requires: hypothesis\n", + "# Symbol presence is checked via AST parse — modules are NOT imported/executed.\n", + "from __future__ import annotations\n", + "import ast\n", + "import pathlib\n", "from hypothesis import strategies as st\n", - "import importlib.util\n", + "\n", + "def _ast_func_names(module_path: str) -> set[str]:\n", + " src = pathlib.Path(module_path).read_text(encoding='utf-8')\n", + " tree = ast.parse(src)\n", + " return {\n", + " n.name\n", + " for n in ast.walk(tree)\n", + " if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))\n", + " }\n", ] idx = 0 blocks = [b.strip() for b in txt.split("\n---\n") if b.strip()] @@ -145,13 +170,8 @@ def stub_tests(path: str) -> str: continue mod = m.group(1).strip().replace("\\", "/") is_virtual = mod.startswith(("ROUTE::", "PROTO::")) - if not is_virtual: - spec_line = ( - "spec = importlib.util.spec_from_file_location(" f"'m_{idx}', '{mod}')" - ) - out.append(spec_line) - out.append(f"m_{idx} = importlib.util.module_from_spec(spec)") - out.append(f"spec.loader.exec_module(m_{idx}) # type: ignore\n") + # Escape for embedding in generated source string literals. + mod_lit = mod.replace("\\", "\\\\").replace("'", "\\'") for fn_block in FN_BLOCK_RE.finditer(b): fn = fn_block.group(1) body = fn_block.group(2) @@ -162,23 +182,21 @@ def stub_tests(path: str) -> str: meta = json.loads(req_m.group(1)) expr = _strategy_expr(meta) out.append(f"req_strategy_{idx}_{fn} = {expr}") - except Exception: + except (json.JSONDecodeError, TypeError, ValueError, KeyError): pass if resp_m: try: meta = json.loads(resp_m.group(1)) expr = _strategy_expr(meta) out.append(f"resp_strategy_{idx}_{fn} = {expr}") - except Exception: + except (json.JSONDecodeError, TypeError, ValueError, KeyError): pass if not is_virtual: out.append( - ( - f"def test_{idx}_{fn}():\n" - f" # TODO: fill inputs;\n" - f" # assert pre/post conditions\n" - f" assert hasattr(m_{idx}, '{fn}')\n" - ) + f"def test_{idx}_{fn}():\n" + f" # TODO: fill inputs; assert pre/post conditions\n" + f" names = _ast_func_names('{mod_lit}')\n" + f" assert '{fn}' in names\n" ) idx += 1 return "\n".join(out) + "\n" @@ -191,7 +209,7 @@ def _sanitize_identifier(s: str) -> str: return s.strip("_") -def _choose_2xx_response(op: Dict[str, Any]) -> Dict[str, Any]: +def _choose_2xx_response(op: dict[str, Any]) -> dict[str, Any]: responses = op.get("responses") or {} if not isinstance(responses, dict): return {} @@ -212,17 +230,21 @@ def from_openapi(openapi_path: str) -> str: return "" # optional validation (non-fatal) try: - if check_openapi_schema is not None and OAS30Validator is not None and isinstance(data, dict): + if ( + check_openapi_schema is not None + and OAS30Validator is not None + and isinstance(data, dict) + ): check_openapi_schema(OAS30Validator, data) - except Exception: + except (TypeError, ValueError, KeyError, AttributeError): pass paths = (data or {}).get("paths", {}) or {} # components not currently used - def extract_schema_meta(schema: Dict[str, Any]) -> Dict[str, Any]: + def extract_schema_meta(schema: dict[str, Any]) -> dict[str, Any]: if not isinstance(schema, dict): return {} - out: Dict[str, Any] = {} + out: dict[str, Any] = {} for k in [ "type", "format", @@ -314,13 +336,20 @@ def from_proto(proto_path: str) -> str: def lean_stubs(contracts_yaml: str, out_dir: str) -> int: + """Emit Lean *scaffold* stubs only (``Prop := True`` / ``by trivial``). + + These files are not meant to be compiled or proven; they mark contract + coverage for presence checks. Real ``lake build`` is out of scope. + """ txt = _read(contracts_yaml) blocks = [b.strip() for b in txt.split("\n---\n") if b.strip()] + os.makedirs(out_dir, exist_ok=True) count = 0 for b in blocks: header = [ "import Std.Data", - f"-- auto-generated from {contracts_yaml}", + f"-- auto-generated scaffold from {contracts_yaml}", + "-- Presence-only stubs: Prop := True / by trivial (not a real proof).", "namespace Contracts\n", ] mod_m = re.search(r"module:\s*(.+)", b) @@ -339,7 +368,7 @@ def lean_stubs(contracts_yaml: str, out_dir: str) -> int: lines.append(theorem_sig) fname = os.path.join(out_dir, f"invariants_{mod_name}.lean") with open(fname, "w", encoding="utf-8") as f: - f.write("\\n".join(lines)) + f.write("\n".join(lines)) count += 1 return count @@ -373,16 +402,16 @@ def report_json(path: str): return {"issues": issues} -def compose(paths: List[str]) -> str: +def compose(paths: list[str]) -> str: """Compose multiple contract YAML files into a single YAML string. Keeps first occurrence order; de-duplicates functions per module by name. """ module_to_fns = OrderedDict() - modules_order: List[str] = [] + modules_order: list[str] = [] for p in paths: try: txt = _read(p) - except Exception: + except OSError: continue for b in _parse_blocks(txt): m = re.search(r"module:\s*(.+)", b) @@ -399,26 +428,26 @@ def compose(paths: List[str]) -> str: body = fn_block.group(2) if fn not in fn_map: # Store exact body including newlines/indentation - fn_map[fn] = [f" {fn}:"] + [ - line for line in body.splitlines() if line.strip() - ] + fn_map[fn] = [f" {fn}:"] + [line for line in body.splitlines() if line.strip()] # else: duplicate; keep first - sections: List[str] = [] + sections: list[str] = [] for mod in modules_order: parts = [f"module: {mod}", "functions:"] - for fn, lines in module_to_fns[mod].items(): + for _fn, lines in module_to_fns[mod].items(): parts.extend(lines) sections.append("\n".join(parts)) return ("\n---\n".join(sections) + "\n") if sections else "" -def verify_lean(contracts_yaml: str, lean_dir: str) -> Dict[str, Any]: - """Verify that Lean stubs exist for each function in contracts. - Returns a JSON-serializable summary; does not attempt to compile Lean. +def verify_lean(contracts_yaml: str, lean_dir: str) -> dict[str, Any]: + """Presence-only check that Lean scaffold stubs exist for each contract function. + + Does **not** compile Lean or run ``lake build``. Returns a JSON-serializable + summary with ``mode: presence-only``. """ txt = _read(contracts_yaml) blocks = _parse_blocks(txt) - expected: List[str] = [] + expected: list[str] = [] for b in blocks: mod_m = re.search(r"module:\s*(.+)", b) raw_mod = mod_m.group(1).strip() if mod_m else "Module" @@ -432,7 +461,7 @@ def verify_lean(contracts_yaml: str, lean_dir: str) -> Dict[str, Any]: ): fn = _sanitize_identifier(fn_m.group(1)) expected.append(f"theorem invariant_{mod_name}__{fn}_holds") - found: List[str] = [] + found: list[str] = [] try: for r, _, files in os.walk(lean_dir): for f in files: @@ -442,10 +471,11 @@ def verify_lean(contracts_yaml: str, lean_dir: str) -> Dict[str, Any]: for e in expected: if e in content: found.append(e) - except Exception: + except OSError: pass missing = [e for e in expected if e not in set(found)] return { + "mode": "presence-only", "modules_total": len(blocks), "functions_total": len(expected), "invariants_found": len(set(found)), diff --git a/cli/ucli/contracts/lang_strategies.py b/cli/ucli/contracts/lang_strategies.py index b5f12c6..b570e01 100644 --- a/cli/ucli/contracts/lang_strategies.py +++ b/cli/ucli/contracts/lang_strategies.py @@ -1,5 +1,4 @@ import re -from typing import List, Tuple def strategy_for_java_type(t: str) -> str: @@ -24,7 +23,7 @@ def strategy_for_java_type(t: str) -> str: return "st.none()" -def infer_java_method_args(src: str) -> List[Tuple[str, str]]: +def infer_java_method_args(src: str) -> list[tuple[str, str]]: # naive regex for: public static Return foo(Type a, Type2 b) m = re.search(r"\b(\w+)\s*\(([^)]*)\)", src) if not m: @@ -32,7 +31,7 @@ def infer_java_method_args(src: str) -> List[Tuple[str, str]]: params = m.group(2).strip() if not params: return [] - out: List[Tuple[str, str]] = [] + out: list[tuple[str, str]] = [] for part in params.split(","): part = part.strip() pm = re.match(r"([\w<>?, ]+)\s+(\w+)$", part) @@ -64,7 +63,7 @@ def strategy_for_csharp_type(t: str) -> str: return "st.none()" -def infer_csharp_method_args(src: str) -> List[Tuple[str, str]]: +def infer_csharp_method_args(src: str) -> list[tuple[str, str]]: # naive: public static Return Foo(Type a, Type2 b) m = re.search(r"\b(\w+)\s*\(([^)]*)\)", src) if not m: @@ -72,7 +71,7 @@ def infer_csharp_method_args(src: str) -> List[Tuple[str, str]]: params = m.group(2).strip() if not params: return [] - out: List[Tuple[str, str]] = [] + out: list[tuple[str, str]] = [] for part in params.split(","): part = part.strip() pm = re.match(r"([\w<>?, ]+)\s+(\w+)$", part) diff --git a/cli/ucli/dashboard/build.py b/cli/ucli/dashboard/build.py index 1d42457..96dd2a8 100644 --- a/cli/ucli/dashboard/build.py +++ b/cli/ucli/dashboard/build.py @@ -1,18 +1,31 @@ -import json, os -from typing import Dict, Any +import json +import os -def build_dashboard(paths: Dict[str,str]) -> str: - repo = json.load(open(paths.get('repo',''), 'r', encoding='utf-8')) if os.path.exists(paths.get('repo','')) else {} - lens = json.load(open(paths.get('lens',''), 'r', encoding='utf-8')) if os.path.exists(paths.get('lens','')) else {} - bounds = json.load(open(paths.get('bounds',''), 'r', encoding='utf-8')) if os.path.exists(paths.get('bounds','')) else {} + +def build_dashboard(paths: dict[str, str]) -> str: + repo = ( + json.load(open(paths.get("repo", ""), encoding="utf-8")) + if os.path.exists(paths.get("repo", "")) + else {} + ) + lens = ( + json.load(open(paths.get("lens", ""), encoding="utf-8")) + if os.path.exists(paths.get("lens", "")) + else {} + ) + bounds = ( + json.load(open(paths.get("bounds", ""), encoding="utf-8")) + if os.path.exists(paths.get("bounds", "")) + else {} + ) lines = ["# Understanding Dashboard", ""] lines.append(f"- Functions mapped: {len(repo.get('functions', {}))}") lines.append(f"- Lens nodes: {len(lens.get('functions', {}))}") - rt = lens.get('runtime', {}).get('hit_count', 0) + rt = lens.get("runtime", {}).get("hit_count", 0) lines.append(f"- Runtime hits: {rt}") lines.append("") lines.append("## Boundaries") - for k in ('openapi_paths','proto_rpcs','sql_tables'): + for k in ("openapi_paths", "proto_rpcs", "sql_tables"): if bounds.get(k): lines.append(f"- {k}: {len(bounds[k])}") lines.append("") diff --git a/cli/ucli/glossary/build.py b/cli/ucli/glossary/build.py index 0daae8a..d4dad86 100644 --- a/cli/ucli/glossary/build.py +++ b/cli/ucli/glossary/build.py @@ -1,22 +1,28 @@ -import os, re -from typing import List +import os +import re + + +def _read(path): + return open(path, encoding="utf-8", errors="ignore").read() -def _read(path): return open(path, 'r', encoding='utf-8', errors='ignore').read() def build_glossary(root: str = ".") -> str: - terms: List[str] = [] + terms: list[str] = [] # OpenAPI paths and schema names - for r, d, files in os.walk(root): + for r, _d, files in os.walk(root): for f in files: - if f.endswith(('.yml', '.yaml')) and 'openapi' in _read(os.path.join(r,f))[:200].lower(): - data = _read(os.path.join(r,f)) + if ( + f.endswith((".yml", ".yaml")) + and "openapi" in _read(os.path.join(r, f))[:200].lower() + ): + data = _read(os.path.join(r, f)) terms += re.findall(r"\n\s{0,6}(/[^\s:]+):", data) terms += re.findall(r"\bcomponents:\s*\n(?:.|\n)*?schemas:\s*\n((?:.|\n)*)", data) # Proto messages and services - for r, d, files in os.walk(root): + for r, _d, files in os.walk(root): for f in files: - if f.endswith('.proto'): - data = _read(os.path.join(r,f)) + if f.endswith(".proto"): + data = _read(os.path.join(r, f)) terms += re.findall(r"message\s+(\w+)", data) terms += re.findall(r"service\s+(\w+)", data) # Clean up and unique @@ -25,7 +31,14 @@ def build_glossary(root: str = ".") -> str: if isinstance(t, str): flat.append(t.strip()) uniq = sorted(set([t for t in flat if t and len(t) < 100])) - md = ["# Domain Glossary", ""] + md = [ + "# Domain Glossary", + "", + "_Scaffold generated from OpenAPI paths / schema names and protobuf", + "messages/services. Entries are **term inventory only** — definitions", + "are not inferred. Replace scaffold notes with real domain language._", + "", + ] for t in uniq: - md.append(f"- **{t}** — TODO: define") + md.append(f"- **{t}** — _(scaffold: definition not generated)_") return "\n".join(md) + "\n" diff --git a/cli/ucli/graph/graph.py b/cli/ucli/graph/graph.py index 4c2289c..0836c80 100644 --- a/cli/ucli/graph/graph.py +++ b/cli/ucli/graph/graph.py @@ -1,12 +1,12 @@ def write_dot(system_map: dict, out_path: str): - fns = system_map.get('functions', {}) - with open(out_path, 'w', encoding='utf-8') as f: - f.write('digraph G {\n') - for qname, meta in fns.items(): - safe = qname.replace('"', '\"') + fns = system_map.get("functions", {}) + with open(out_path, "w", encoding="utf-8") as f: + f.write("digraph G {\n") + for qname, _meta in fns.items(): + safe = qname.replace('"', '"') f.write(f' "{safe}";\n') for qname, meta in fns.items(): - caller = qname.replace('"', '\"') - for callee in meta.get('calls', []): + caller = qname.replace('"', '"') + for callee in meta.get("calls", []): f.write(f' "{caller}" -> "{callee}";\n') - f.write('}\n') + f.write("}\n") diff --git a/cli/ucli/lens/ingest.py b/cli/ucli/lens/ingest.py index 8d60eaa..5eff3c6 100644 --- a/cli/ucli/lens/ingest.py +++ b/cli/ucli/lens/ingest.py @@ -1,14 +1,16 @@ -import re, json +import re + def seeds_from_github_log(path: str): - txt = open(path, 'r', encoding='utf-8', errors='ignore').read() + txt = open(path, encoding="utf-8", errors="ignore").read() # pytest short tb lines: "file.py:123: in func" files = re.findall(r"([A-Za-z0-9_./-]+\.py):\d+", txt) funcs = re.findall(r"in\s+([A-Za-z_][A-Za-z0-9_]*)\s*$", txt, flags=re.M) return sorted(list(set(files + funcs))) + def seeds_from_jira(path: str): - txt = open(path, 'r', encoding='utf-8', errors='ignore').read() + txt = open(path, encoding="utf-8", errors="ignore").read() # naive: capture code blocks and file.py:line occurrences files = re.findall(r"([A-Za-z0-9_./-]+\.py):\d+", txt) funcs = re.findall(r"([A-Za-z_][A-Za-z0-9_]*)\(", txt) diff --git a/cli/ucli/lens/lens.py b/cli/ucli/lens/lens.py index 8f04305..d62b267 100644 --- a/cli/ucli/lens/lens.py +++ b/cli/ucli/lens/lens.py @@ -1,35 +1,78 @@ -import re, json, math -from typing import Dict, Any, List, Set, Tuple +import re +from typing import Any +from ucli.analyzers.map_meta import annotate_lens_from_map -def _extract_candidates(issue_text: str) -> List[str]: - files = re.findall(r"[\w./-]+\.py", issue_text) + +def _extract_candidates(issue_text: str) -> list[str]: + files = re.findall(r"[\w./-]+\.(?:py|js|mjs|cjs|ts|tsx|jsx)", issue_text) funcs = re.findall(r"([A-Za-z_][A-Za-z0-9_]*)\(", issue_text) return list(set(files + funcs)) -def _neighbors(functions: Dict[str, Any], seed_keys: Set[str], hops: int = 2) -> Dict[str, Any]: +def _seed_matches(q: str, meta: dict[str, Any], seed_keys: set[str], *, exact: bool) -> bool: + """True when qname/meta matches any seed token. + + Default (exact=False): substring match — ``tok in q`` (historical behavior; + ``run`` matches ``pkg:Foo.run`` and also ``pkg:runner``). + + exact=True: full qname, local name after ``:``, or ``simple_name`` equality only. + """ + if exact: + local = q.rsplit(":", 1)[-1] + simple = meta.get("simple_name") or local.rsplit(".", 1)[-1] + return q in seed_keys or local in seed_keys or simple in seed_keys + return any(tok in q for tok in seed_keys) + + +def _callee_matches_qn(callee: str, q2: str) -> bool: + """Match a call token to a function qname without inventing edges. + + Prefers exact qname equality (JS same-file qualified calls), then local-name + suffix / containment heuristics used historically for Python bare names. + """ + if callee == q2: + return True + if ":" in callee: + # Already qualified — only exact identity (no substring invention). + return False + return q2.endswith(":" + callee) or callee in q2 + + +def _neighbors( + functions: dict[str, Any], + seed_keys: set[str], + hops: int = 2, + *, + exact: bool = False, +) -> dict[str, Any]: keep = set() - for q in functions: - if any(tok in q for tok in seed_keys): + for q, meta in functions.items(): + if _seed_matches(q, meta, seed_keys, exact=exact): keep.add(q) for _ in range(hops): new = set(keep) for q, meta in functions.items(): if q in keep: for callee in meta.get("calls", []): + if callee in functions: + new.add(callee) + continue for q2 in functions: - if q2.endswith(":" + callee) or callee in q2: + if _callee_matches_qn(str(callee), q2): new.add(q2) else: - if any(c in q for k in keep for c in functions[k].get("calls", [])): - new.add(q) + for k in keep: + for c in functions[k].get("calls", []): + if c == q or _callee_matches_qn(str(c), q): + new.add(q) + break keep = new return {q: functions[q] for q in keep} -def lens_from_issue(issue_md_path: str, repo_map: Dict[str, Any], hops: int = 2) -> Dict[str, Any]: - text = open(issue_md_path, "r", encoding="utf-8").read() +def lens_from_issue(issue_md_path: str, repo_map: dict[str, Any], hops: int = 2) -> dict[str, Any]: + text = open(issue_md_path, encoding="utf-8").read() seeds = _extract_candidates(text) if not seeds: fns = repo_map.get("functions", {}) @@ -37,16 +80,27 @@ def lens_from_issue(issue_md_path: str, repo_map: Dict[str, Any], hops: int = 2) seeds = [k for k, _ in ranked] seed_keys = set(seeds) lens_funcs = _neighbors(repo_map.get("functions", {}), seed_keys, hops=hops) - return {"lens": {"seeds": list(seed_keys)}, "functions": lens_funcs} + lens = {"lens": {"seeds": list(seed_keys)}, "functions": lens_funcs} + return annotate_lens_from_map(lens, repo_map) -def lens_from_seeds(seeds: List[str], repo_map: Dict[str, Any], hops: int = 2) -> Dict[str, Any]: +def lens_from_seeds( + seeds: list[str], + repo_map: dict[str, Any], + hops: int = 2, + *, + exact: bool = False, +) -> dict[str, Any]: seed_keys = set(seeds) - lens_funcs = _neighbors(repo_map.get("functions", {}), seed_keys, hops=hops) - return {"lens": {"seeds": list(seed_keys)}, "functions": lens_funcs} + lens_funcs = _neighbors(repo_map.get("functions", {}), seed_keys, hops=hops, exact=exact) + lens = { + "lens": {"seeds": list(seed_keys), "exact_match": exact}, + "functions": lens_funcs, + } + return annotate_lens_from_map(lens, repo_map) -def merge_trace_into_lens(lens: Dict[str, Any], trace: Dict[str, Any]) -> Dict[str, Any]: +def merge_trace_into_lens(lens: dict[str, Any], trace: dict[str, Any]) -> dict[str, Any]: hits = {e.get("func") for e in trace.get("events", []) if "func" in e} fns = lens.get("functions", {}) for q in fns: @@ -57,7 +111,7 @@ def merge_trace_into_lens(lens: Dict[str, Any], trace: Dict[str, Any]) -> Dict[s return lens -def rank_by_error_proximity(lens: Dict[str, Any]) -> None: +def rank_by_error_proximity(lens: dict[str, Any]) -> None: # score nodes: seed match + runtime hit + distance to seeds (by name similarity) seeds = set(lens.get("lens", {}).get("seeds", [])) fns = lens.get("functions", {}) @@ -76,16 +130,19 @@ def rank_by_error_proximity(lens: Dict[str, Any]) -> None: meta["error_proximity"] = round(score, 3) -def write_tour_md(lens: Dict[str, Any]) -> str: +def write_tour_md(lens: dict[str, Any]) -> str: fns = lens.get("functions", {}) ranked = sorted(fns.items(), key=lambda kv: kv[1].get("error_proximity", 0.0), reverse=True) files = [] - for q, meta in ranked: + for _q, meta in ranked: if meta.get("file"): files.append(meta["file"]) files = list(dict.fromkeys(files)) top3 = files[:3] seeds = lens.get("lens", {}).get("seeds", []) + fidelity = lens.get("lens", {}).get("fidelity") or lens.get("analyzer_fidelity") + complexity_note = lens.get("lens", {}).get("complexity_note") + language = lens.get("language") or lens.get("lens", {}).get("language") out = ["# 10-minute Task Tour", "", "## Start here (3 files)"] out += [f"1. `{p}`" for p in top3] out += [ @@ -95,26 +152,55 @@ def write_tour_md(lens: Dict[str, Any]) -> str: "- Side effects documented (files, network, globals)", ] out += ["", "## Minimal fixture", "Run the generated fixture (if present) to hit the hot path."] - out += ["", "## Seeds", ", ".join(seeds)] + out += ["", "## Seeds", ", ".join(str(s) for s in seeds)] + # Honesty: JS/TS lenses are structural only — no Python tour_run parity. + out += ["", "## Map fidelity"] + if language: + out.append(f"- Language: `{language}`") + if isinstance(fidelity, dict): + out.append(f"- Analyzer fidelity: `{fidelity}`") + elif fidelity: + out.append(f"- Analyzer fidelity: `{fidelity}`") + if complexity_note: + out.append(f"- {complexity_note}") + if language in {"javascript", "typescript"} or ( + isinstance(fidelity, str) and fidelity == "best-effort" + ): + out.append( + "- Runtime `u trace` / `u tour_run` are Python-only; this tour is a reading plan " + "from a best-effort map, not CLI runtime parity." + ) + elif language in {"go", "java", "rust", "csharp"}: + out.append( + f"- Language `{language}`: structural/best-effort map only; " + "`u trace` / `u tour_run` remain Python-only." + ) + elif language == "mixed": + out.append( + "- Mixed map: Python subsets may support `u trace`; other languages are " + "map/reading-plan only." + ) return "\n".join(out) -def explain_node(qname: str, lens: Dict[str, Any], repo_map: Dict[str, Any]) -> Dict[str, Any]: +def explain_node(qname: str, lens: dict[str, Any], repo_map: dict[str, Any]) -> dict[str, Any]: fns = repo_map.get("functions", {}) seeds = set(lens.get("lens", {}).get("seeds", [])) meta = lens.get("functions", {}).get(qname, {}) name = qname.split(":")[-1] - # callers/callees from repo map (by suffix match heuristic) - callers: List[str] = [] - callees: List[str] = [] + # callers/callees from repo map (prefer exact qnames on JS maps) + callers: list[str] = [] + callees: list[str] = [] for caller_q, m in fns.items(): for c in m.get("calls", []): - if qname.endswith(":" + c) or name == c: + if c == qname or _callee_matches_qn(str(c), qname) or name == c: callers.append(caller_q) for c in fns.get(qname, {}).get("calls", []): - # find qualified + if c in fns: + callees.append(c) + continue for cand in fns: - if cand.endswith(":" + c) or c in cand: + if _callee_matches_qn(str(c), cand): callees.append(cand) callers = list(dict.fromkeys(callers)) callees = list(dict.fromkeys(callees)) diff --git a/cli/ucli/main.py b/cli/ucli/main.py index 02670bc..7540156 100644 --- a/cli/ucli/main.py +++ b/cli/ucli/main.py @@ -1,10 +1,5 @@ -from typing import Optional, List, Dict, Any -import json -import os import pathlib -import subprocess import sys -import shutil # The package is `cli.ucli` but modules use absolute `ucli.*` imports; ensure `cli/` # is on sys.path so those resolve when the app is run from an installed console script. @@ -18,74 +13,14 @@ if callable(reconf): try: reconf(encoding="utf-8") - except Exception: + except (OSError, ValueError, AttributeError): pass -import socket + import typer -from rich import print from rich.console import Console -from rich.progress import ( - Progress, - SpinnerColumn, - TextColumn, - BarColumn, - TimeElapsedColumn, - TaskProgressColumn, -) -from rich.panel import Panel -from rich.table import Table -from rich.prompt import Prompt, Confirm, IntPrompt -from rich.layout import Layout -from rich.live import Live -from rich.text import Text -from rich.align import Align -from rich.columns import Columns -from rich.markdown import Markdown -from rich.status import Status -from rich.syntax import Syntax -from rich.tree import Tree -import time -from datetime import datetime - -from ucli.analyzers.python_analyzer import build_python_map -from ucli.graph.graph import write_dot -from ucli.report.report import make_report_md - -from ucli.lens.lens import ( - lens_from_issue, - lens_from_seeds, - merge_trace_into_lens, - write_tour_md, - rank_by_error_proximity, - explain_node, -) -from ucli.lens.ingest import seeds_from_github_log, seeds_from_jira -from ucli.trace.pytrace import run_callable_with_trace -from ucli.trace.pytrace import analyze_errors_static -from ucli.boundaries.scan import scan_boundaries -from ucli.contracts.contracts import ( - init_contracts, - check_contracts, - stub_tests, - from_openapi, - from_proto, - lean_stubs, - report_json, - compose, - verify_lean, -) -from ucli.packs.pack import create_pack -from ucli.visual.delta import lens_delta_svg -from ucli.config import load_config -from ucli.config import load_preset -from ucli.config import validate_config_dict -from ucli.glossary.build import build_glossary -from ucli.dashboard.build import build_dashboard -from ucli.pack.publish import make_pack -from ucli.metrics.ttu import record as ttu_record -from ucli.metrics.ttu import weekly_report as ttu_weekly -from ucli.metrics.analytics import get_tracker, track_event, get_dashboard_data +from ucli.commands.init_ops import show_banner, show_welcome_message +from ucli.commands.typer_groups import register_all_groups app = typer.Typer( help=( @@ -96,4709 +31,31 @@ no_args_is_help=True, ) +register_all_groups(app) -@app.callback() + +@app.callback(invoke_without_command=True) def main( - version: bool = typer.Option(False, "--version", "-v", help="Show version and exit"), + ctx: typer.Context, + version: bool = typer.Option( + False, "--version", "-V", is_eager=True, help="Show version and exit" + ), help: bool = typer.Option(False, "--help", "-h", help="Show help and exit"), ): - """🧠 Understand First CLI - Accelerate code understanding with intelligent analysis.""" + """Understand First CLI - Accelerate code understanding with intelligent analysis.""" console = Console() if version: - console.print("Understand First CLI v1.0.0", style="bold cyan") - raise typer.Exit(0) - - if help: - show_banner() - show_welcome_message() - - # Show quick reference - help_table = Table(title="📚 Quick Reference", show_header=True, header_style="bold cyan") - help_table.add_column("Command", style="cyan", width=20) - help_table.add_column("Description", style="white") - help_table.add_column("Example", style="dim") - - help_table.add_row("init", "Initialize new project", "u init my-project") - help_table.add_row("scan", "Analyze codebase", "u scan --interactive") - help_table.add_row( - "lens from-seeds", - "Create focused analysis", - "u lens from-seeds --map out.json --seed main", - ) - help_table.add_row("tour", "Generate understanding tour", "u tour lens.json") - help_table.add_row("report", "Create detailed report", "u report out.json") - help_table.add_row("diff", "Compare analysis results", "u diff old.json new.json") - - console.print(help_table) - console.print( - "\n[yellow]💡 Tip:[/yellow] Use [cyan]--help[/cyan] with any command for detailed information" - ) - raise typer.Exit(0) - - -def show_banner(): - """Display the Understand First banner.""" - banner = """ -╔═══════════════════════════════════════════════════════════════╗ -║ ║ -║ 🧠 Understand First CLI ║ -║ ════════════════════════════ ║ -║ ║ -║ Accelerate code understanding with intelligent analysis ║ -║ and guided tours for faster, safer development. ║ -║ ║ -╚═══════════════════════════════════════════════════════════════╝ -""" - console = Console() - console.print(banner, style="cyan bold") - - -def show_welcome_message(): - """Display welcome message with quick start guide.""" - console = Console() - - welcome_panel = Panel( - "[bold cyan]Welcome to Understand First![/bold cyan]\n\n" - "This CLI helps you understand codebases faster through:\n" - "• 🔍 Intelligent code analysis and mapping\n" - "• 🎯 Focused lenses for specific tasks\n" - "• 🗺️ Interactive understanding tours\n" - "• 📊 Complexity and dependency insights\n" - "• 🔗 Contract and boundary analysis\n\n" - "[bold]Quick Start:[/bold]\n" - "1. [cyan]u init[/cyan] - Initialize a new project\n" - "2. [cyan]u scan[/cyan] - Analyze your codebase\n" - "3. [cyan]u lens from-seeds[/cyan] - Create focused analysis\n" - "4. [cyan]u tour[/cyan] - Generate understanding tour", - title="🚀 Getting Started", - border_style="green", - ) - - console.print(welcome_panel) - - -@app.command() -def init( - path: str = typer.Argument(".", help="Path to initialize the project"), - wizard: bool = typer.Option(True, "--wizard/--no-wizard", help="Run interactive setup wizard"), - force: bool = typer.Option(False, "--force", help="Overwrite existing configuration"), -): - """Initialize a new Understand First project with interactive wizard.""" - console = Console() + console.print("Understand First CLI v0.3.0", style="bold cyan") + raise typer.Exit() - # Show banner on first run - if not pathlib.Path(path).exists() or force: + if ctx.invoked_subcommand is None or help: show_banner() show_welcome_message() - - try: - project_path = pathlib.Path(path) - - if not project_path.exists(): - console.print(f"[yellow]Creating project directory: {project_path}[/yellow]") - project_path.mkdir(parents=True, exist_ok=True) - - # Check for existing config - config_file = project_path / "understand-first.json" - if config_file.exists() and not force: - if not Confirm.ask(f"Configuration file already exists. Overwrite?"): - console.print("[yellow]Initialization cancelled.[/yellow]") - return - - if wizard: - console.print("\n[bold cyan]🔧 Project Setup Wizard[/bold cyan]\n") - - # Project information - project_name = Prompt.ask("Project name", default=project_path.name) - project_description = Prompt.ask("Project description", default="") - - # Language selection - language = Prompt.ask( - "Primary programming language", - choices=["python", "javascript", "typescript", "java", "go", "rust", "other"], - default="python", - ) - - # Analysis preferences - console.print("\n[bold]Analysis Configuration:[/bold]") - include_tests = Confirm.ask("Include test files in analysis?", default=True) - include_vendors = Confirm.ask("Include vendor/dependency files?", default=False) - complexity_threshold = IntPrompt.ask("Complexity threshold (1-10)", default=5) - - # Output preferences - console.print("\n[bold]Output Configuration:[/bold]") - output_dir = Prompt.ask("Output directory", default="maps") - auto_generate_tours = Confirm.ask("Auto-generate tours after analysis?", default=True) - - # Create configuration - config = { - "project": { - "name": project_name, - "description": project_description, - "language": language, - "path": str(project_path), - }, - "analysis": { - "include_tests": include_tests, - "include_vendors": include_vendors, - "complexity_threshold": complexity_threshold, - "exclude_patterns": [ - "*.pyc", - "__pycache__", - ".git", - "node_modules", - "venv", - ".env", - ], - }, - "output": { - "directory": output_dir, - "auto_generate_tours": auto_generate_tours, - "formats": ["json", "dot", "md"], - }, - "created_at": datetime.now().isoformat(), - "version": "1.0", - } - - # Save configuration - with open(config_file, "w", encoding="utf-8") as f: - json.dump(config, f, indent=2) - - # Create directory structure - output_path = project_path / output_dir - output_path.mkdir(exist_ok=True) - - # Create sample files - sample_code = project_path / "sample.py" - if not sample_code.exists(): - with open(sample_code, "w", encoding="utf-8") as f: - f.write( - """# Sample Python code for analysis -def hello_world(): - \"\"\"A simple greeting function.\"\"\" - print("Hello, World!") - -def calculate_sum(a, b): - \"\"\"Calculate the sum of two numbers.\"\"\" - return a + b - -class Calculator: - \"\"\"A simple calculator class.\"\"\" - - def __init__(self): - self.history = [] - - def add(self, x, y): - result = x + y - self.history.append(f"{x} + {y} = {result}") - return result - -if __name__ == "__main__": - hello_world() - calc = Calculator() - result = calc.add(5, 3) - print(f"Result: {result}") -""" - ) - - # Show completion message - success_panel = Panel( - f"[green]✓ Project initialized successfully![/green]\n\n" - f"📁 [bold]Project:[/bold] {project_name}\n" - f"📝 [bold]Description:[/bold] {project_description}\n" - f"🐍 [bold]Language:[/bold] {language}\n" - f"📊 [bold]Output:[/bold] {output_dir}/\n\n" - f"🎯 [bold]Next steps:[/bold]\n" - f" • Run [cyan]u scan[/cyan] to analyze your codebase\n" - f" • Run [cyan]u lens from-seeds --seed hello_world[/cyan] to create a lens\n" - f" • Run [cyan]u tour[/cyan] to generate an understanding tour", - title="🎉 Project Ready!", - border_style="green", - ) - - console.print(success_panel) - - # Offer to run first analysis - if Confirm.ask("\nWould you like to run your first analysis now?"): - console.print("\n[bold cyan]🔍 Running initial analysis...[/bold cyan]\n") - # Call the scan function with current path - scan(str(project_path), o=str(output_path / "out.json"), verbose=True) - else: - # Non-wizard mode - create minimal config - config = { - "project": {"name": project_path.name, "path": str(project_path)}, - "analysis": {"include_tests": True, "complexity_threshold": 5}, - "output": {"directory": "maps"}, - "created_at": datetime.now().isoformat(), - "version": "1.0", - } - - with open(config_file, "w", encoding="utf-8") as f: - json.dump(config, f, indent=2) - - console.print(f"[green]✓ Minimal configuration created at {config_file}[/green]") - - except KeyboardInterrupt: - console.print("\n[yellow]Initialization cancelled by user.[/yellow]") - raise typer.Exit(0) - except Exception as e: - console.print(f"[red]Error during initialization:[/red] {str(e)}") - raise typer.Exit(1) - - -@app.command() -def scan( - path: str = typer.Argument("."), - o: str = typer.Option("maps/out.json", "--output", "-o"), - verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable verbose output"), - debug: bool = typer.Option(False, "--debug", help="Enable debug mode with tracebacks"), - interactive: bool = typer.Option( - False, "--interactive", "-i", help="Interactive scan with guided options" - ), -): - """Scan codebase and build repository map with enhanced progress tracking.""" - console = Console() - - try: - p = pathlib.Path(path) - - # Validate input path - if not p.exists(): - console.print(f"[red]❌ Error:[/red] Path '{path}' does not exist") - console.print( - f"[yellow]💡 Tip:[/yellow] Use [cyan]u init[/cyan] to set up a new project" - ) - raise typer.Exit(1) - - # Check if it's a valid code directory - if ( - not any(p.glob("**/*.py")) - and not any(p.glob("**/*.js")) - and not any(p.glob("**/*.ts")) - ): - console.print( - f"[yellow]⚠️ Warning:[/yellow] No supported source files found in '{path}'" - ) - if not Confirm.ask("Continue anyway?"): - raise typer.Exit(0) - - # Interactive mode enhancements - if interactive: - console.print("\n[bold cyan]🔧 Interactive Scan Configuration[/bold cyan]\n") - - # Ask for scan options - include_tests = Confirm.ask("Include test files?", default=True) - include_docs = Confirm.ask("Include documentation files?", default=True) - max_depth = IntPrompt.ask("Maximum analysis depth (1-10)", default=5) - - # Update output path if needed - custom_output = Prompt.ask("Output file path", default=o) - if custom_output != o: - o = custom_output - - # Load project config if available - config_file = p / "understand-first.json" - config = {} - if config_file.exists(): - try: - with open(config_file, "r", encoding="utf-8") as f: - config = json.load(f) - if verbose: - console.print(f"[dim]Loaded configuration from {config_file}[/dim]") - except Exception as e: - console.print(f"[yellow]Warning:[/yellow] Could not load config: {e}") - - # Show scan configuration - config_table = Table(title="📋 Scan Configuration", show_header=False, box=None) - config_table.add_column("Setting", style="cyan", width=20) - config_table.add_column("Value", style="white") - config_table.add_row("Target Path", str(p.absolute())) - config_table.add_row("Output File", o) - config_table.add_row("Verbose Mode", "✅ Enabled" if verbose else "❌ Disabled") - config_table.add_row("Debug Mode", "✅ Enabled" if debug else "❌ Disabled") - - if config: - config_table.add_row("Project", config.get("project", {}).get("name", "Unknown")) - config_table.add_row( - "Language", config.get("project", {}).get("language", "Auto-detect") - ) - - console.print(config_table) - console.print() - - # Enhanced progress tracking with better error handling - with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - BarColumn(), - TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), - TimeElapsedColumn(), - console=console, - ) as progress: - - # Task 1: Discover files - discover_task = progress.add_task("🔍 Discovering source files...", total=100) - - try: - # Count Python files for better progress estimation - py_files = list(p.rglob("*.py")) - total_files = len(py_files) - - if total_files == 0: - console.print( - "[yellow]⚠️ No Python files found. Trying other languages...[/yellow]" - ) - # Try other file types - js_files = list(p.rglob("*.js")) - ts_files = list(p.rglob("*.ts")) - total_files = len(js_files) + len(ts_files) - - if total_files == 0: - console.print("[red]❌ No supported source files found![/red]") - raise typer.Exit(1) - - progress.update(discover_task, advance=100) - - if verbose: - console.print(f"[dim]Found {total_files} source files[/dim]") - - except Exception as e: - progress.update(discover_task, advance=100) - if debug: - console.print(f"[red]File discovery error:[/red] {e}") - else: - console.print( - "[yellow]⚠️ File discovery encountered issues, continuing...[/yellow]" - ) - - # Task 2: Parse and analyze - parse_task = progress.add_task("🧠 Analyzing code structure...", total=100) - - try: - # Actually build the map with better error handling - if verbose: - console.print("[dim]Building Python map with enhanced analysis...[/dim]") - - result = build_python_map(p) - - if not result or not result.get("functions"): - console.print("[yellow]⚠️ Analysis completed but no functions found[/yellow]") - console.print( - "[dim]This might be normal for empty or non-Python projects[/dim]" - ) - - progress.update(parse_task, advance=100) - - except Exception as e: - progress.update(parse_task, advance=100) - console.print(f"[red]❌ Analysis failed:[/red] {str(e)}") - if debug: - import traceback - - console.print(f"[dim]{traceback.format_exc()}[/dim]") - raise typer.Exit(1) - - # Task 3: Write output - write_task = progress.add_task("💾 Writing analysis results...", total=100) - - try: - output_path = pathlib.Path(o) - output_path.parent.mkdir(parents=True, exist_ok=True) - - with open(output_path, "w", encoding="utf-8") as f: - json.dump(result, f, indent=2) - - progress.update(write_task, advance=100) - - if verbose: - console.print(f"[dim]Results written to {output_path.absolute()}[/dim]") - - except Exception as e: - progress.update(write_task, advance=100) - console.print(f"[red]❌ Failed to write output:[/red] {str(e)}") - raise typer.Exit(1) - - # Show results summary with enhanced information - functions_count = len(result.get("functions", {})) - modules_count = len( - set(func.get("file", "") for func in result.get("functions", {}).values()) - ) - - # Calculate complexity metrics - complexities = [func.get("complexity", 0) for func in result.get("functions", {}).values()] - avg_complexity = sum(complexities) / len(complexities) if complexities else 0 - high_complexity_count = sum(1 for c in complexities if c > 7) - - results_panel = Panel( - f"[green]✅ Scan completed successfully![/green]\n\n" - f"📊 [bold]Analysis Results:[/bold]\n" - f" • Functions analyzed: {functions_count}\n" - f" • Modules processed: {modules_count}\n" - f" • Average complexity: {avg_complexity:.1f}\n" - f" • High complexity functions: {high_complexity_count}\n" - f" • Output file: [cyan]{o}[/cyan]\n\n" - f"🎯 [bold]Recommended Next Steps:[/bold]\n" - f" • [cyan]u lens from-seeds --map {o} --seed [/cyan] - Create focused analysis\n" - f" • [cyan]u tour {o.replace('.json', '_lens.json')}[/cyan] - Generate understanding tour\n" - f" • [cyan]u report {o}[/cyan] - Generate detailed report\n" - f" • [cyan]u map {o}[/cyan] - Create visual graph", - title="🎉 Scan Results", - border_style="green", - ) - - console.print(results_panel) - - # Show warnings if any - if high_complexity_count > 0: - console.print( - f"\n[yellow]⚠️ Found {high_complexity_count} high-complexity functions (>7)[/yellow]" - ) - console.print("[dim]Consider reviewing these for refactoring opportunities[/dim]") - - # Track TTU metric - try: - ttu_record( - "scan_completed", - { - "path": str(p), - "functions_count": functions_count, - "modules_count": modules_count, - "avg_complexity": avg_complexity, - "high_complexity_count": high_complexity_count, - }, - ) - except Exception as e: - if debug: - console.print(f"[dim]Failed to track metrics: {e}[/dim]") - - except KeyboardInterrupt: - console.print("\n[yellow]⚠️ Scan cancelled by user[/yellow]") - raise typer.Exit(0) - except Exception as e: - console.print(f"[red]❌ Error during scan:[/red] {str(e)}") - - # Provide helpful error messages - if "Permission denied" in str(e): - console.print( - "[yellow]💡 Tip:[/yellow] Check file permissions or try running with different privileges" - ) - elif "No such file" in str(e): - console.print("[yellow]💡 Tip:[/yellow] Verify the path exists and is accessible") - elif "JSON" in str(e): - console.print("[yellow]💡 Tip:[/yellow] Check if the output directory is writable") - - if debug: - import traceback - - console.print(f"[dim]{traceback.format_exc()}[/dim]") - raise typer.Exit(1) - - -@app.command() -def map(json_path: str, o: str = typer.Option("maps", "--output", "-o")): - """Generate DOT graph visualization from analysis results.""" - console = Console() - - try: - if not pathlib.Path(json_path).exists(): - console.print(f"[red]❌ Error:[/red] File '{json_path}' does not exist") - raise typer.Exit(1) - - with open(json_path, "r", encoding="utf-8") as f: - data = json.load(f) - - os.makedirs(o, exist_ok=True) - dot_path = os.path.join(o, pathlib.Path(json_path).stem + ".dot") - write_dot(data, dot_path) - - console.print(f"[green]✅ Wrote[/green] {dot_path}") - - except Exception as e: - console.print(f"[red]❌ Error generating map:[/red] {str(e)}") - raise typer.Exit(1) - - -@app.command() -def diff( - old_map: str = typer.Argument(..., help="Path to old analysis results"), - new_map: str = typer.Argument(..., help="Path to new analysis results"), - o: str = typer.Option( - "maps/delta.svg", "--output", "-o", help="Output file for delta visualization" - ), - verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable verbose output"), - policy_check: bool = typer.Option( - True, "--policy-check/--no-policy-check", help="Run policy compliance checks" - ), -): - """Compare two analysis results and generate delta visualization with policy checks.""" - console = Console() - - try: - # Validate input files - for file_path in [old_map, new_map]: - if not pathlib.Path(file_path).exists(): - console.print(f"[red]❌ Error:[/red] File '{file_path}' does not exist") - raise typer.Exit(1) - - console.print("[bold cyan]🔄 Comparing analysis results...[/bold cyan]\n") - - # Load both maps - with open(old_map, "r", encoding="utf-8") as f: - old_data = json.load(f) - with open(new_map, "r", encoding="utf-8") as f: - new_data = json.load(f) - - # Calculate differences - old_functions = old_data.get("functions", {}) - new_functions = new_data.get("functions", {}) - - added_functions = set(new_functions.keys()) - set(old_functions.keys()) - removed_functions = set(old_functions.keys()) - set(new_functions.keys()) - modified_functions = set() - - for func_name in set(old_functions.keys()) & set(new_functions.keys()): - if old_functions[func_name] != new_functions[func_name]: - modified_functions.add(func_name) - - # Show diff summary - diff_table = Table(title="📊 Analysis Delta Summary", show_header=False, box=None) - diff_table.add_column("Metric", style="cyan", width=20) - diff_table.add_column("Count", style="white") - diff_table.add_column("Status", style="white") - - diff_table.add_row("Added Functions", str(len(added_functions)), "🟢") - diff_table.add_row("Removed Functions", str(len(removed_functions)), "🔴") - diff_table.add_row("Modified Functions", str(len(modified_functions)), "🟡") - diff_table.add_row("Total Functions (Old)", str(len(old_functions)), "📊") - diff_table.add_row("Total Functions (New)", str(len(new_functions)), "📊") - - console.print(diff_table) - console.print() - - # Policy compliance checks - if policy_check: - console.print("[bold cyan]🔍 Running Policy Compliance Checks...[/bold cyan]\n") - - policy_violations = [] - - # Check for complexity increases - complexity_increases = [] - for func_name in modified_functions: - old_complexity = old_functions[func_name].get("complexity", 0) - new_complexity = new_functions[func_name].get("complexity", 0) - if new_complexity > old_complexity + 2: # Significant increase - complexity_increases.append( - { - "function": func_name, - "old": old_complexity, - "new": new_complexity, - "increase": new_complexity - old_complexity, - } - ) - - if complexity_increases: - policy_violations.append( - { - "type": "Complexity Increase", - "count": len(complexity_increases), - "severity": ( - "high" - if any(c["increase"] > 5 for c in complexity_increases) - else "medium" - ), - } - ) - - if verbose: - console.print( - "[yellow]⚠️ Functions with significant complexity increases:[/yellow]" - ) - for violation in complexity_increases[:5]: # Show first 5 - console.print( - f" • {violation['function']}: {violation['old']} → {violation['new']} (+{violation['increase']})" - ) - - # Check for new high-complexity functions - new_high_complexity = [ - name for name in added_functions if new_functions[name].get("complexity", 0) > 8 - ] - - if new_high_complexity: - policy_violations.append( - { - "type": "New High Complexity", - "count": len(new_high_complexity), - "severity": "high", - } - ) - - if verbose: - console.print("[yellow]⚠️ New high-complexity functions:[/yellow]") - for func_name in new_high_complexity[:5]: - complexity = new_functions[func_name].get("complexity", 0) - console.print(f" • {func_name}: complexity {complexity}") - - # Policy summary - if policy_violations: - console.print("\n[bold red]🚨 Policy Violations Detected:[/bold red]") - for violation in policy_violations: - severity_icon = "🔴" if violation["severity"] == "high" else "🟡" - console.print( - f" {severity_icon} {violation['type']}: {violation['count']} instances" - ) - - if any(v["severity"] == "high" for v in policy_violations): - console.print( - "\n[yellow]⚠️ High severity violations detected. Review required.[/yellow]" - ) - else: - console.print("[green]✅ No policy violations detected[/green]") - - # Generate delta visualization - console.print(f"\n[bold cyan]🎨 Generating delta visualization...[/bold cyan]") - - try: - os.makedirs(pathlib.Path(o).parent, exist_ok=True) - svg = lens_delta_svg(old_map, new_map) - with open(o, "w", encoding="utf-8") as df: - df.write(svg) - console.print(f"[green]✅ Delta visualization written to {o}[/green]") - except Exception as e: - console.print(f"[yellow]⚠️ Could not generate visualization: {e}[/yellow]") - - # Summary and recommendations - summary_panel = Panel( - f"[bold]Delta Analysis Complete[/bold]\n\n" - f"📈 [bold]Changes:[/bold] {len(added_functions)} added, {len(removed_functions)} removed, {len(modified_functions)} modified\n" - f"🔍 [bold]Policy Check:[/bold] {'✅ Passed' if not policy_violations else '❌ Violations detected'}\n" - f"📊 [bold]Output:[/bold] {o}\n\n" - f"🎯 [bold]Recommendations:[/bold]\n" - f" • Review complexity increases for refactoring opportunities\n" - f" • Consider breaking down high-complexity functions\n" - f" • Update documentation for modified functions", - title="🎉 Delta Analysis Results", - border_style="green" if not policy_violations else "yellow", - ) - - console.print(summary_panel) - - # Track metrics - try: - ttu_record( - "diff_completed", - { - "old_functions": len(old_functions), - "new_functions": len(new_functions), - "added": len(added_functions), - "removed": len(removed_functions), - "modified": len(modified_functions), - "policy_violations": len(policy_violations) if policy_check else 0, - }, - ) - except Exception as e: - if verbose: - console.print(f"[dim]Failed to track metrics: {e}[/dim]") - - # Exit with appropriate code - if policy_violations and any(v["severity"] == "high" for v in policy_violations): - raise typer.Exit(1) # High severity violations - - except KeyboardInterrupt: - console.print("\n[yellow]⚠️ Diff cancelled by user[/yellow]") - raise typer.Exit(0) - except Exception as e: - console.print(f"[red]❌ Error during diff:[/red] {str(e)}") - raise typer.Exit(1) - - -@app.command() -def report(json_path: str, o: str = typer.Option("maps", "--output", "-o")): - with open(json_path, "r", encoding="utf-8") as f: - data = json.load(f) - os.makedirs(o, exist_ok=True) - md_path = os.path.join(o, pathlib.Path(json_path).stem + ".md") - with open(md_path, "w", encoding="utf-8") as f: - f.write(make_report_md(data)) - print(f"[green]Wrote[/green] {md_path}") - - -# Lenses -lens_app = typer.Typer(help="Task lenses") -app.add_typer(lens_app, name="lens") - - -@lens_app.command("from-issue") -def lens_from_issue_cmd( - issue_md: str, - map: str = typer.Option(..., "--map"), - o: str = typer.Option("maps/lens.json", "--output", "-o"), -): - with open(map, "r", encoding="utf-8") as f: - repo_map = json.load(f) - lens = lens_from_issue(issue_md, repo_map) - rank_by_error_proximity(lens) - os.makedirs(pathlib.Path(o).parent, exist_ok=True) - with open(o, "w", encoding="utf-8") as f: - json.dump(lens, f, indent=2) - print(f"[green]Wrote[/green] {o}") - - -@lens_app.command("from-seeds") -def lens_from_seeds_cmd( - seed: List[str] = typer.Option([], "--seed"), - map: str = typer.Option(..., "--map"), - o: str = typer.Option("maps/lens.json", "--output", "-o"), - interactive: bool = typer.Option( - False, "--interactive", "-i", help="Interactive seed selection" - ), - verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable verbose output"), -): - """Create understanding lens from seed functions with enhanced TUI.""" - console = Console() - - try: - # Load map data - with open(map, "r", encoding="utf-8") as f: - repo_map = json.load(f) - - seeds = seed.copy() if seed else [] - - if interactive: - # Interactive seed selection - console.print( - Panel( - "[bold blue]🔍 Interactive Seed Selection[/bold blue]\n" - "Select functions to include in your understanding lens.\n" - "This will help focus your analysis on specific areas of the codebase.", - title="Lens Creation", - border_style="blue", - ) - ) - - functions = list(repo_map.get("functions", {}).keys()) - if not functions: - console.print("[red]No functions found in map. Run 'u scan' first.[/red]") - raise typer.Exit(1) - - # Show available functions in a table - func_table = Table(title="Available Functions", show_header=True) - func_table.add_column("Index", style="cyan", width=6) - func_table.add_column("Function", style="white") - func_table.add_column("File", style="dim") - func_table.add_column("Complexity", style="yellow") - - for i, func_name in enumerate(functions, 1): - func_data = repo_map["functions"][func_name] - complexity = func_data.get("complexity", 0) - file_path = func_data.get("file", "unknown") - func_table.add_row(str(i), func_name, file_path, str(complexity)) - - console.print(func_table) - console.print() - - # Interactive selection - while True: - choice = Prompt.ask("Select function index (or 'done' to finish)", default="done") - - if choice.lower() == "done": - break - - try: - index = int(choice) - 1 - if 0 <= index < len(functions): - selected_func = functions[index] - if selected_func not in seeds: - seeds.append(selected_func) - console.print(f"[green]✓ Added[/green] {selected_func}") - else: - console.print(f"[yellow]⚠ Already selected[/yellow] {selected_func}") - else: - console.print("[red]Invalid index. Please try again.[/red]") - except ValueError: - console.print("[red]Please enter a valid number or 'done'.[/red]") - - if not seeds: - console.print("[yellow]No seeds selected. Using first 5 functions.[/yellow]") - seeds = functions[:5] # Limit to first 5 for performance - - # Show lens configuration - config_table = Table(title="Lens Configuration", show_header=False) - config_table.add_column("Setting", style="cyan") - config_table.add_column("Value", style="white") - config_table.add_row("Map File", map) - config_table.add_row("Seeds", f"{len(seeds)} selected") - config_table.add_row("Output File", o) - - console.print(config_table) console.print() - - # Create lens with progress - with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - BarColumn(), - TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), - TimeElapsedColumn(), - console=console, - ) as progress: - - lens_task = progress.add_task("Creating understanding lens...", total=100) - progress.update(lens_task, advance=30) - - lens = lens_from_seeds(seeds, repo_map) - rank_by_error_proximity(lens) - - progress.update(lens_task, advance=70) - - # Write output - os.makedirs(pathlib.Path(o).parent, exist_ok=True) - with open(o, "w", encoding="utf-8") as f: - json.dump(lens, f, indent=2) - - progress.update(lens_task, advance=100) - - # Show results - lens_functions = len(lens.get("functions", {})) - - results_panel = Panel( - f"[green]✓ Lens created successfully![/green]\n\n" - f"📊 [bold]Lens Summary:[/bold]\n" - f" • Functions in lens: {lens_functions}\n" - f" • Seeds used: {len(seeds)}\n" - f" • Output file: {o}\n\n" - f"🎯 [bold]Next steps:[/bold]\n" - f" • Run [cyan]u tour {o}[/cyan] to generate an understanding tour\n" - f" • Run [cyan]u trace module [/cyan] to add runtime traces", - title="Lens Creation Results", - border_style="green", - ) - - console.print(results_panel) - - # Track TTU metric - ttu_record( - "lens_created", - { - "seeds_count": len(seeds), - "lens_functions": lens_functions, - "interactive": interactive, - }, - ) - - except Exception as e: - console.print(f"[red]Error creating lens:[/red] {str(e)}") - if verbose: - import traceback - - console.print(f"[dim]{traceback.format_exc()}[/dim]") - raise typer.Exit(1) - - -@lens_app.command("merge-trace") -def lens_merge_trace_cmd( - lens_json: str, - trace_json: str, - o: str = typer.Option("maps/lens_merged.json", "--output", "-o"), -): - with open(lens_json, "r", encoding="utf-8") as f: - lens = json.load(f) - with open(trace_json, "r", encoding="utf-8") as f: - trace = json.load(f) - merged = merge_trace_into_lens(lens, trace) - rank_by_error_proximity(merged) - os.makedirs(pathlib.Path(o).parent, exist_ok=True) - with open(o, "w", encoding="utf-8") as f: - json.dump(merged, f, indent=2) - print(f"[green]Wrote[/green] {o}") - - -@lens_app.command("preset") -def lens_preset_cmd( - label: str, - map: str = typer.Option(..., "--map"), - o: str = typer.Option("maps/lens.json", "--output", "-o"), -): - cfg = load_config() - seeds = load_preset(label) - hops = cfg.get("hops", 2) - with open(map, "r", encoding="utf-8") as f: - repo_map = json.load(f) - lens = lens_from_seeds(seeds, repo_map, hops=hops) - with open(o, "w", encoding="utf-8") as wf: - json.dump(lens, wf, indent=2) - print(f"[green]Lens written[/green] {o}") - - -@lens_app.command("ingest-github") -def ingest_github(log_path: str): - seeds = seeds_from_github_log(log_path) - print(json.dumps({"seeds": seeds}, indent=2)) - - -@lens_app.command("ingest-jira") -def ingest_jira(jira_path: str): - seeds = seeds_from_jira(jira_path) - print(json.dumps({"seeds": seeds}, indent=2)) - - -@app.command() -def tour(lens_json: str, o: str = typer.Option("tours/tour.md", "--output", "-o")): - with open(lens_json, "r", encoding="utf-8") as f: - lens = json.load(f) - os.makedirs(pathlib.Path(o).parent, exist_ok=True) - md = write_tour_md(lens) - with open(o, "w", encoding="utf-8") as f: - f.write(md) - print(f"[green]Wrote[/green] {o}") - - -@app.command() -def tour_run(lens_json: str, fixtures_dir: str = typer.Option("fixtures", "--fixtures", "-f")): - """Attempt to run a minimal fixture and verify runtime hits align with the lens.""" - # Try running a default fixture file if present, else generate a minimal one. - fixture = os.path.join(fixtures_dir, "fixture_hot_path.py") - if not os.path.exists(fixture): - print("[yellow]No fixture found; cannot run tour.[/yellow]") - raise typer.Exit(1) - # Execute fixture in a subprocess; success if return code 0 - rc = subprocess.call(["python", fixture]) - if rc != 0: - print("[red]Fixture failed.[/red]") - raise typer.Exit(rc) - print("[green]Fixture ran successfully.[/green]") - - -# Trace -trace_app = typer.Typer(help="Runtime tracing (Python demo)") -app.add_typer(trace_app, name="trace") - - -@trace_app.command("module") -def trace_module( - pyfile: str, - func: str, - a: Optional[str] = None, - b: Optional[str] = None, - o: str = typer.Option("traces/trace.json", "--output", "-o"), -): - os.makedirs(pathlib.Path(o).parent, exist_ok=True) - data = run_callable_with_trace(pyfile, func, a, b) - with open(o, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2) - print(f"[green]Wrote[/green] {o}") - - -@trace_app.command("errors") -def trace_errors(pyfile: str, json_out: bool = typer.Option(True, "--json/--no-json")): - data = analyze_errors_static(pyfile) - if json_out: - print(json.dumps(data, indent=2)) - else: - print("Raises:") - for r in data.get("raises", []): - print(f" line {r.get('line')}: {r.get('exc')}") - print("Catches:") - for c in data.get("catches", []): - print(f" line {c.get('line')}: {c.get('catch')}") - - -# Boundaries -bound_app = typer.Typer(help="Boundary scanners") -app.add_typer(bound_app, name="boundaries") - - -@bound_app.command("scan") -def boundaries_scan( - path: str = typer.Argument("."), - o: str = typer.Option("maps/boundaries.json", "--output", "-o"), -): - result = scan_boundaries(path) - os.makedirs(pathlib.Path(o).parent, exist_ok=True) - with open(o, "w", encoding="utf-8") as f: - json.dump(result, f, indent=2) - print(f"[green]Wrote[/green] {o}") - - -# Contracts -contracts_app = typer.Typer(help="Contracts") -app.add_typer(contracts_app, name="contracts") - - -@contracts_app.command("init") -def contracts_init( - path: str = typer.Argument("."), - o: str = typer.Option("contracts/contracts.yaml", "--output", "-o"), -): - os.makedirs(pathlib.Path(o).parent, exist_ok=True) - txt = init_contracts(path) - with open(o, "w", encoding="utf-8") as f: - f.write(txt) - print(f"[green]Wrote[/green] {o}") - - -@contracts_app.command("check") -def contracts_check(path: str): - ok, report = check_contracts(path) - print(report) - if not ok: - raise typer.Exit(1) - - -@contracts_app.command("stub-tests") -def contracts_stub(path: str, o: str = typer.Option("tests/test_contracts.py", "--output", "-o")): - os.makedirs(pathlib.Path(o).parent, exist_ok=True) - txt = stub_tests(path) - with open(o, "w", encoding="utf-8") as f: - f.write(txt) - print(f"[green]Wrote[/green] {o}") - - -@contracts_app.command("from-openapi") -def contracts_from_openapi( - path: str, - o: str = typer.Option("contracts/contracts_from_openapi.yaml", "--output", "-o"), -): - txt = from_openapi(path) - os.makedirs(pathlib.Path(o).parent, exist_ok=True) - with open(o, "w", encoding="utf-8") as f: - f.write(txt) - print(f"[green]Wrote[/green] {o}") - - -@contracts_app.command("from-proto") -def contracts_from_proto( - path: str, - o: str = typer.Option("contracts/contracts_from_proto.yaml", "--output", "-o"), -): - txt = from_proto(path) - os.makedirs(pathlib.Path(o).parent, exist_ok=True) - with open(o, "w", encoding="utf-8") as f: - f.write(txt) - print(f"[green]Wrote[/green] {o}") - - -@contracts_app.command("lean-stubs") -def contracts_lean_stubs( - contracts_yaml: str, o: str = typer.Option("contracts/lean/", "--output-dir", "-o") -): - os.makedirs(o, exist_ok=True) - count = lean_stubs(contracts_yaml, o) - print(f"[green]Wrote[/green] {count} Lean stub(s) to {o}") - - -@contracts_app.command("compose") -def contracts_compose( - i: List[str] = typer.Option([], "--input", "-i", help="Contract YAML input paths"), - o: str = typer.Option("contracts/contracts.yaml", "--output", "-o"), -): - if not i: - print("[red]No input files provided[/red]") - raise typer.Exit(1) - txt = compose(i) - os.makedirs(pathlib.Path(o).parent, exist_ok=True) - with open(o, "w", encoding="utf-8") as f: - f.write(txt) - print(f"[green]Wrote[/green] {o}") - - -@contracts_app.command("verify-lean") -def contracts_verify_lean( - contracts_yaml: str = typer.Argument(...), - lean_dir: str = typer.Option("contracts/lean", "--lean-dir", "-l"), - json_out: bool = typer.Option(False, "--json"), -): - data = verify_lean(contracts_yaml, lean_dir) - if json_out: - print(json.dumps(data, indent=2)) - else: - print(f"modules: {data.get('modules_total')}\n" f"functions: {data.get('functions_total')}") - missing = data.get("missing_invariants", []) - if missing: - print("missing invariants:") - for m in missing: - print(f" - {m}") - if data.get("missing_invariants"): - raise typer.Exit(1) - - -# Packs -pack_app = typer.Typer(help="Understanding Packs") -app.add_typer(pack_app, name="pack") - - -@pack_app.command("create") -def pack_create( - lens: str = typer.Option(..., "--lens"), - tour: str = typer.Option(..., "--tour"), - contracts: str = typer.Option(..., "--contracts"), - o: str = typer.Option("packs/pack.zip", "--output", "-o"), -): - os.makedirs(pathlib.Path(o).parent, exist_ok=True) - create_pack(lens, tour, contracts, o) - print(f"[green]Wrote[/green] {o}") - - -# Visual -visual_app = typer.Typer(help="Visualization utilities") -app.add_typer(visual_app, name="visual") - - -@visual_app.command("delta") -def visual_delta( - old_lens: str, - new_lens: str, - o: str = typer.Option("maps/delta.svg", "--output", "-o"), -): - svg = lens_delta_svg(old_lens, new_lens) - os.makedirs(pathlib.Path(o).parent, exist_ok=True) - with open(o, "w", encoding="utf-8") as f: - f.write(svg) - print(f"[green]Wrote[/green] {o}") - - -# C-02: Diff Map Delta Helper Functions -def _load_analysis_data(source: str) -> Optional[Dict[str, Any]]: - """Load analysis data from file or git commit.""" - if os.path.exists(source): - # Load from file - with open(source, "r") as f: - return json.load(f) - else: - # Try to load from git commit - try: - result = subprocess.run( - ["git", "show", f"{source}:.understand-first/analysis.json"], - capture_output=True, - text=True, - check=True, - ) - return json.loads(result.stdout) - except (subprocess.CalledProcessError, json.JSONDecodeError): - return None - - -def _generate_delta_analysis(before: Dict[str, Any], after: Dict[str, Any]) -> Dict[str, Any]: - """Generate comprehensive delta analysis between two analysis results.""" - delta = { - "summary": { - "before_functions": len(before.get("functions", {})), - "after_functions": len(after.get("functions", {})), - "added_functions": 0, - "removed_functions": 0, - "modified_functions": 0, - "complexity_change": 0, - "side_effects_added": 0, - "side_effects_removed": 0, - }, - "added": [], - "removed": [], - "modified": [], - "complexity_changes": [], - "side_effect_changes": [], - "hot_path_changes": [], - "timestamp": datetime.now().isoformat(), - } - - before_funcs = before.get("functions", {}) - after_funcs = after.get("functions", {}) - - # Find added and removed functions - before_names = set(before_funcs.keys()) - after_names = set(after_funcs.keys()) - - added_names = after_names - before_names - removed_names = before_names - after_names - common_names = before_names & after_names - - delta["summary"]["added_functions"] = len(added_names) - delta["summary"]["removed_functions"] = len(removed_names) - - # Process added functions - for name in added_names: - func_data = after_funcs[name] - delta["added"].append( - { - "name": name, - "file": func_data.get("file", ""), - "line": func_data.get("line", 0), - "complexity": func_data.get("complexity", 0), - "side_effects": func_data.get("side_effects", []), - "is_hot_path": func_data.get("is_hot_path", False), - } - ) - - # Process removed functions - for name in removed_names: - func_data = before_funcs[name] - delta["removed"].append( - { - "name": name, - "file": func_data.get("file", ""), - "line": func_data.get("line", 0), - "complexity": func_data.get("complexity", 0), - "side_effects": func_data.get("side_effects", []), - "is_hot_path": func_data.get("is_hot_path", False), - } - ) - - # Process modified functions - for name in common_names: - before_func = before_funcs[name] - after_func = after_funcs[name] - - changes = {} - if before_func.get("complexity", 0) != after_func.get("complexity", 0): - changes["complexity"] = { - "before": before_func.get("complexity", 0), - "after": after_func.get("complexity", 0), - "delta": after_func.get("complexity", 0) - before_func.get("complexity", 0), - } - delta["complexity_changes"].append( - { - "name": name, - "file": after_func.get("file", ""), - "line": after_func.get("line", 0), - **changes["complexity"], - } - ) - - before_side_effects = set(before_func.get("side_effects", [])) - after_side_effects = set(after_func.get("side_effects", [])) - - if before_side_effects != after_side_effects: - added_side_effects = after_side_effects - before_side_effects - removed_side_effects = before_side_effects - after_side_effects - - changes["side_effects"] = { - "added": list(added_side_effects), - "removed": list(removed_side_effects), - } - - delta["side_effect_changes"].append( - { - "name": name, - "file": after_func.get("file", ""), - "line": after_func.get("line", 0), - **changes["side_effects"], - } - ) - - delta["summary"]["side_effects_added"] += len(added_side_effects) - delta["summary"]["side_effects_removed"] += len(removed_side_effects) - - if before_func.get("is_hot_path", False) != after_func.get("is_hot_path", False): - changes["hot_path"] = { - "before": before_func.get("is_hot_path", False), - "after": after_func.get("is_hot_path", False), - } - delta["hot_path_changes"].append( - { - "name": name, - "file": after_func.get("file", ""), - "line": after_func.get("line", 0), - **changes["hot_path"], - } - ) - - if changes: - delta["modified"].append( - { - "name": name, - "file": after_func.get("file", ""), - "line": after_func.get("line", 0), - "changes": changes, - } - ) - delta["summary"]["modified_functions"] += 1 - - # Calculate overall complexity change - before_complexity = sum(func.get("complexity", 0) for func in before_funcs.values()) - after_complexity = sum(func.get("complexity", 0) for func in after_funcs.values()) - delta["summary"]["complexity_change"] = after_complexity - before_complexity - - return delta - - -def _check_policy_compliance(delta: Dict[str, Any]) -> List[Dict[str, Any]]: - """Check delta against policy rules and return violations.""" - violations = [] - - # Policy: No new high-complexity functions (>10) - for func in delta["added"]: - if func["complexity"] > 10: - violations.append( - { - "type": "high_complexity_added", - "severity": "warning", - "function": func["name"], - "file": func["file"], - "line": func["line"], - "complexity": func["complexity"], - "message": f"New function '{func['name']}' has high complexity ({func['complexity']})", - } - ) - - # Policy: No new unmanaged side effects - for func in delta["added"]: - unmanaged_side_effects = [ - se for se in func["side_effects"] if se not in ["logging", "metrics"] - ] - if unmanaged_side_effects: - violations.append( - { - "type": "unmanaged_side_effects_added", - "severity": "error", - "function": func["name"], - "file": func["file"], - "line": func["line"], - "side_effects": unmanaged_side_effects, - "message": f"New function '{func['name']}' has unmanaged side effects: {', '.join(unmanaged_side_effects)}", - } - ) - - # Policy: Significant complexity increase (>5 points) - for change in delta["complexity_changes"]: - if change["delta"] > 5: - violations.append( - { - "type": "complexity_increase", - "severity": "warning", - "function": change["name"], - "file": change["file"], - "line": change["line"], - "before": change["before"], - "after": change["after"], - "delta": change["delta"], - "message": f"Function '{change['name']}' complexity increased by {change['delta']} points", - } - ) - - # Policy: New side effects added to existing functions - for change in delta["side_effect_changes"]: - if change["added"]: - violations.append( - { - "type": "side_effects_added", - "severity": "warning", - "function": change["name"], - "file": change["file"], - "line": change["line"], - "added_side_effects": change["added"], - "message": f"Function '{change['name']}' now has additional side effects: {', '.join(change['added'])}", - } - ) - - return violations - - -def _generate_markdown_delta(delta: Dict[str, Any], violations: List[Dict[str, Any]]) -> str: - """Generate markdown delta report.""" - md = [] - md.append("# Analysis Delta Report") - md.append(f"Generated: {delta['timestamp']}") - md.append("") - - # Summary - summary = delta["summary"] - md.append("## Summary") - md.append(f"- **Functions Added**: {summary['added_functions']}") - md.append(f"- **Functions Removed**: {summary['removed_functions']}") - md.append(f"- **Functions Modified**: {summary['modified_functions']}") - md.append(f"- **Complexity Change**: {summary['complexity_change']:+d}") - md.append(f"- **Side Effects Added**: {summary['side_effects_added']}") - md.append(f"- **Side Effects Removed**: {summary['side_effects_removed']}") - md.append("") - - # Policy violations - if violations: - md.append("## ⚠️ Policy Violations") - for violation in violations: - severity_icon = "🚨" if violation["severity"] == "error" else "⚠️" - md.append(f"### {severity_icon} {violation['type'].replace('_', ' ').title()}") - md.append(f"**Function**: `{violation['function']}`") - md.append(f"**File**: `{violation['file']}:{violation['line']}`") - md.append(f"**Message**: {violation['message']}") - md.append("") - - # Added functions - if delta["added"]: - md.append("## Added Functions") - for func in delta["added"]: - md.append(f"- `{func['name']}` in `{func['file']}:{func['line']}`") - md.append(f" - Complexity: {func['complexity']}") - if func["side_effects"]: - md.append(f" - Side Effects: {', '.join(func['side_effects'])}") - if func["is_hot_path"]: - md.append(f" - 🔥 Hot Path") - md.append("") - - # Removed functions - if delta["removed"]: - md.append("## Removed Functions") - for func in delta["removed"]: - md.append(f"- `{func['name']}` in `{func['file']}:{func['line']}`") - md.append("") - - # Modified functions - if delta["modified"]: - md.append("## Modified Functions") - for func in delta["modified"]: - md.append(f"- `{func['name']}` in `{func['file']}:{func['line']}`") - for change_type, change_data in func["changes"].items(): - if change_type == "complexity": - md.append( - f" - Complexity: {change_data['before']} → {change_data['after']} ({change_data['delta']:+d})" - ) - elif change_type == "side_effects": - if change_data["added"]: - md.append(f" - Added Side Effects: {', '.join(change_data['added'])}") - if change_data["removed"]: - md.append(f" - Removed Side Effects: {', '.join(change_data['removed'])}") - md.append("") - - return "\n".join(md) - - -def _generate_html_delta(delta: Dict[str, Any], violations: List[Dict[str, Any]]) -> str: - """Generate HTML delta report.""" - html = [] - html.append("") - html.append("Analysis Delta Report") - html.append("") - - html.append("

Analysis Delta Report

") - html.append(f"

Generated: {delta['timestamp']}

") - - # Summary - summary = delta["summary"] - html.append("
") - html.append("

Summary

") - html.append(f"

Functions Added: {summary['added_functions']}

") - html.append(f"

Functions Removed: {summary['removed_functions']}

") - html.append(f"

Functions Modified: {summary['modified_functions']}

") - html.append(f"

Complexity Change: {summary['complexity_change']:+d}

") - html.append(f"

Side Effects Added: {summary['side_effects_added']}

") - html.append(f"

Side Effects Removed: {summary['side_effects_removed']}

") - html.append("
") - - # Policy violations - if violations: - html.append("

⚠️ Policy Violations

") - for violation in violations: - severity_class = "violation" if violation["severity"] == "error" else "warning" - html.append(f"
") - html.append(f"

{violation['type'].replace('_', ' ').title()}

") - html.append(f"

Function: {violation['function']}

") - html.append( - f"

File: {violation['file']}:{violation['line']}

" - ) - html.append(f"

Message: {violation['message']}

") - html.append("
") - - # Added functions - if delta["added"]: - html.append("

Added Functions

") - for func in delta["added"]: - html.append(f"
") - html.append( - f"{func['name']} in {func['file']}:{func['line']}" - ) - html.append(f"
    ") - html.append(f"
  • Complexity: {func['complexity']}
  • ") - if func["side_effects"]: - html.append(f"
  • Side Effects: {', '.join(func['side_effects'])}
  • ") - if func["is_hot_path"]: - html.append(f"
  • 🔥 Hot Path
  • ") - html.append(f"
") - html.append("
") - - html.append("") - return "\n".join(html) - - -def _display_delta_summary( - console: Console, delta: Dict[str, Any], violations: List[Dict[str, Any]] -): - """Display a summary of the delta analysis.""" - summary = delta["summary"] - - # Create summary table - table = Table(title="Delta Analysis Summary") - table.add_column("Metric", style="cyan") - table.add_column("Value", style="white") - - table.add_row("Functions Added", str(summary["added_functions"])) - table.add_row("Functions Removed", str(summary["removed_functions"])) - table.add_row("Functions Modified", str(summary["modified_functions"])) - table.add_row("Complexity Change", f"{summary['complexity_change']:+d}") - table.add_row("Side Effects Added", str(summary["side_effects_added"])) - table.add_row("Side Effects Removed", str(summary["side_effects_removed"])) - - console.print(table) - - # Show policy violations - if violations: - console.print(f"\n[red]Policy Violations: {len(violations)}[/red]") - for violation in violations[:5]: # Show first 5 - severity_color = "red" if violation["severity"] == "error" else "yellow" - console.print(f"[{severity_color}]• {violation['message']}[/{severity_color}]") - - if len(violations) > 5: - console.print(f"[dim]... and {len(violations) - 5} more violations[/dim]") - else: - console.print("\n[green]No policy violations detected[/green]") + console.print(ctx.get_help()) + raise typer.Exit() if __name__ == "__main__": app() - - -@app.command() -def glossary(o: str = typer.Option("docs/glossary.md", "--output", "-o")): - os.makedirs(os.path.dirname(o), exist_ok=True) - md = build_glossary(".") - with open(o, "w", encoding="utf-8") as f: - f.write(md) - print(f"[green]Wrote[/green] {o}") - - -@app.command() -def dashboard( - repo: str = typer.Option("maps/repo.json", "--repo"), - lens: str = typer.Option("maps/lens_merged.json", "--lens"), - bounds: str = typer.Option("maps/boundaries.json", "--bounds"), - o: str = typer.Option("docs/understanding-dashboard.md", "--output", "-o"), -): - os.makedirs(os.path.dirname(o), exist_ok=True) - md = build_dashboard({"repo": repo, "lens": lens, "bounds": bounds}) - with open(o, "w", encoding="utf-8") as f: - f.write(md) - print(f"[green]Wrote[/green] {o}") - - -@contracts_app.command("report") -def contracts_report( - path: str, - json_out: bool = typer.Option(False, "--json", help="Emit JSON report to stdout"), -): - data = report_json(path) - if json_out: - print(json.dumps(data, indent=2)) - else: - for i in data.get("issues", []): - print( - f"[{i.get('severity','info')}] " - f"{i.get('module_path')}:{i.get('function')} - {i.get('message')}" - ) - if data.get("issues"): - raise typer.Exit(1) - - -@app.command() -def pack( - publish: bool = typer.Option( - True, "--publish/--no-publish", help="Build pack artifacts (local zip)." - ) -): - if publish: - make_pack("dist") - print("[green]Pack ready in dist/[/green]") - - -@app.command() -def ttu( - event: str = typer.Argument(...), - o: str = typer.Option("docs/ttu.md", "--output", "-o"), -): - if event == "report": - ttu_weekly(o) - print(f"[green]Wrote[/green] {o}") - else: - ttu_record(event) - print(f"[green]Recorded[/green] {event}") - - -@app.command() -def lens_preset( - label: str, - map: str = typer.Option(..., "--map"), - o: str = typer.Option("maps/lens.json", "--output", "-o"), -): - cfg = load_config() - seeds = load_preset(label) - hops = cfg.get("hops", 2) - with open(map, "r", encoding="utf-8") as f: - repo_map = json.load(f) - lens = lens_from_seeds(seeds, repo_map, hops=hops) - with open(o, "w", encoding="utf-8") as wf: - json.dump(lens, wf, indent=2) - print(f"[green]Lens written[/green] {o}") - - -@app.command() -def doctor(): - problems: List[str] = [] - notes: List[str] = [] - - def ok(msg: str): - print(f"[green]OK[/green] {msg}") - - def warn(msg: str, fix: Optional[str] = None): - print(f"[yellow]WARN[/yellow] {msg}") - if fix: - notes.append(f"- {msg} → {fix}") - - def fail(msg: str, fix: Optional[str] = None): - print(f"[red]FAIL[/red] {msg}") - problems.append(msg) - if fix: - notes.append(f"- {msg} → {fix}") - - ok(f"Python {sys.version.split()[0]}") - - try: - r = subprocess.run( - ["node", "-v"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True - ) - if r.returncode == 0: - ok(f"Node {r.stdout.strip()}") - else: - warn( - "Node not found in PATH", - "Install Node 18+ or use Devcontainer/Codespaces", - ) - except Exception: - warn("Node not found in PATH", "Install Node 18+ or use Devcontainer/Codespaces") - - try: - import grpc_tools # type: ignore - - ok("grpc_tools available") - except Exception: - fail("grpc_tools missing", "pip install grpcio-tools") - - def can_bind(port: int) -> bool: - try: - s = socket.socket() - s.bind(("127.0.0.1", port)) - s.close() - return True - except Exception: - return False - - if not can_bind(8000): - warn( - "Port 8000 not available", - "Stop the process using 8000 or change server port", - ) - else: - ok("Port 8000 available") - if not can_bind(50051): - warn( - "Port 50051 not available", - "Stop the process using 50051 or change gRPC port", - ) - else: - ok("Port 50051 available") - - code_bin = shutil.which("code") or shutil.which("code.cmd") - if code_bin: - ok(f"VS Code found at {code_bin}") - else: - warn("VS Code not found", "Install VS Code or use Codespaces") - - try: - test_file = pathlib.Path(".uf_write_test") - test_file.write_text("ok", encoding="utf-8") - test_file.unlink() - ok("Repo write permissions OK") - except Exception: - fail( - "No write permission in repo", - "Check filesystem permissions or workspace settings", - ) - - print("\nNext steps") - doc_url = "https://github.com/your-org/understand-first#readme" - for n in notes: - print(n) - if problems: - print(f"\nSee docs: {doc_url}") - raise typer.Exit(code=1) - else: - print("All checks passed. See docs for advanced setup:") - print(doc_url) - - -@app.command() -def demo(): - try: - txt = from_openapi("examples/apis/petstore-mini.yaml") - os.makedirs("contracts", exist_ok=True) - open("contracts/contracts_from_openapi.yaml", "w", encoding="utf-8").write(txt) - except Exception: - pass - - http_proc = subprocess.Popen([sys.executable, "examples/servers/http_server.py"]) # nosec - try: - os.makedirs("traces", exist_ok=True) - data = run_callable_with_trace("examples/app/hot_path.py", "run_hot_path") - open("traces/tour.json", "w", encoding="utf-8").write(json.dumps(data, indent=2)) - - os.makedirs("maps", exist_ok=True) - repo_map = build_python_map(pathlib.Path("examples/python_toy")) - open("maps/repo.json", "w", encoding="utf-8").write(json.dumps(repo_map, indent=2)) - lens = lens_from_seeds(["compute"], repo_map) - merged = merge_trace_into_lens(lens, data) - rank_by_error_proximity(merged) - open("maps/lens_merged.json", "w", encoding="utf-8").write(json.dumps(merged, indent=2)) - - os.makedirs("tours", exist_ok=True) - open("tours/demo.md", "w", encoding="utf-8").write(write_tour_md(merged)) - os.makedirs("docs", exist_ok=True) - open("docs/understanding-dashboard.md", "w", encoding="utf-8").write( - build_dashboard( - { - "repo": "maps/repo.json", - "lens": "maps/lens_merged.json", - "bounds": "maps/boundaries.json", - } - ) - ) - - url = f"file://{pathlib.Path('tours/demo.md').resolve()}" - print(f"[green]Open tour:[/green] {url}") - finally: - http_proc.terminate() - - -@app.command() -def init( - stack: str = typer.Option("py", "--stack"), - ci: str = typer.Option("github", "--ci"), - wizard: bool = typer.Option(False, "--wizard", help="Interactive configuration wizard"), - tui: bool = typer.Option(False, "--tui", help="Launch interactive TUI mode"), -): - """Initialize understand-first configuration for your project.""" - console = Console() - - if tui: - _launch_tui_mode() - elif wizard: - _run_config_wizard() - else: - _create_basic_config(stack, ci) - - -@app.command() -def diff( - before: str = typer.Argument(..., help="Path to before analysis (JSON file or commit hash)"), - after: str = typer.Argument(..., help="Path to after analysis (JSON file or commit hash)"), - output: str = typer.Option( - "delta.json", "--output", "-o", help="Output file for delta analysis" - ), - format: str = typer.Option( - "json", "--format", "-f", help="Output format: json, markdown, html" - ), - policy_check: bool = typer.Option( - True, "--policy-check", help="Enable policy compliance checking" - ), - verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable verbose output"), -): - """Compare two analysis results and generate delta report with policy checking.""" - console = Console() - - try: - # Load before and after analyses - before_data = _load_analysis_data(before) - after_data = _load_analysis_data(after) - - if not before_data or not after_data: - console.print("[red]Error:[/red] Could not load analysis data") - raise typer.Exit(1) - - # Generate delta analysis - delta_result = _generate_delta_analysis(before_data, after_data) - - # Policy compliance checking - policy_violations = [] - if policy_check: - policy_violations = _check_policy_compliance(delta_result) - - # Generate output in requested format - if format == "markdown": - output_content = _generate_markdown_delta(delta_result, policy_violations) - elif format == "html": - output_content = _generate_html_delta(delta_result, policy_violations) - else: - output_content = json.dumps(delta_result, indent=2) - - # Write output - with open(output, "w") as f: - f.write(output_content) - - # Display summary - _display_delta_summary(console, delta_result, policy_violations) - - # Exit with appropriate code - if policy_violations: - console.print(f"[yellow]Policy violations detected: {len(policy_violations)}[/yellow]") - raise typer.Exit(2) - else: - console.print("[green]Delta analysis completed successfully[/green]") - raise typer.Exit(0) - - except Exception as e: - if verbose: - console.print(f"[red]Error:[/red] {e}") - import traceback - - traceback.print_exc() - else: - console.print(f"[red]Error:[/red] {e}") - raise typer.Exit(1) - - -def _create_basic_config(stack: str, ci: str): - """Create a basic configuration file.""" - os.makedirs(".understand-first.yml".replace(".yml", ""), exist_ok=True) # no-op - open(".understand-first.yml", "w", encoding="utf-8").write( - """hops: 2 -seeds: [] -seeds_for: - bug: [examples/app/hot_path.py] -metrics: - enabled: false -""" - ) - os.makedirs("tours", exist_ok=True) - open("README.md", "a", encoding="utf-8").write( - "\n\n## 10-minute tour\nRun `u scan` then `u demo`.\n" - ) - print("[green]Initialized understand-first config and basics[/green]") - - -def _launch_tui_mode(): - """Launch interactive TUI mode for understand-first.""" - console = Console() - - # Create layout - layout = Layout() - layout.split_column( - Layout(name="header", size=3), Layout(name="main"), Layout(name="footer", size=3) - ) - - layout["main"].split_row(Layout(name="sidebar", size=30), Layout(name="content")) - - # Header - header_panel = Panel( - Align.center("[bold blue]🧠 Understand-First TUI[/bold blue]"), style="blue" - ) - layout["header"].update(header_panel) - - # Footer - footer_text = Text("Press 'q' to quit, 'h' for help, arrow keys to navigate", style="dim") - layout["footer"].update(Panel(Align.center(footer_text), style="dim")) - - # Main TUI loop - with Live(layout, refresh_per_second=4, screen=True) as live: - _run_tui_main_loop(live, layout) - - -def _run_tui_main_loop(live, layout): - """Main TUI event loop.""" - console = Console() - current_view = "dashboard" - - while True: - # Update content based on current view - if current_view == "dashboard": - _render_dashboard(layout) - elif current_view == "scan": - _render_scan_view(layout) - elif current_view == "lens": - _render_lens_view(layout) - elif current_view == "tour": - _render_tour_view(layout) - elif current_view == "config": - _render_config_view(layout) - - # Handle input - try: - key = console.input() - if key.lower() == "q": - break - elif key.lower() == "h": - _show_help(layout) - elif key == "1": - current_view = "dashboard" - elif key == "2": - current_view = "scan" - elif key == "3": - current_view = "lens" - elif key == "4": - current_view = "tour" - elif key == "5": - current_view = "config" - except KeyboardInterrupt: - break - - -def _render_dashboard(layout): - """Render the main dashboard view.""" - # Sidebar - sidebar_content = """ -[bold]Navigation[/bold] -1. Dashboard -2. Scan Code -3. Lens Management -4. Tour Builder -5. Configuration - -[bold]Quick Actions[/bold] -• Press 's' to scan current directory -• Press 'l' to create new lens -• Press 't' to generate tour -• Press 'c' to configure project -""" - layout["sidebar"].update(Panel(sidebar_content, title="Menu", border_style="blue")) - - # Main content - dashboard_content = """ -[bold green]Welcome to Understand-First TUI![/bold green] - -This interactive interface helps you: -• Analyze your codebase structure -• Create focused lenses for specific areas -• Generate understanding tours -• Track Time-to-Understanding metrics - -[bold]Recent Activity[/bold] -• No recent scans found -• No active lenses -• No tours generated - -[bold]Project Status[/bold] -• Configuration: Not initialized -• Last scan: Never -• Active lenses: 0 -• Generated tours: 0 - -[bold]Quick Start[/bold] -1. Press 'c' to configure your project -2. Press 's' to scan your codebase -3. Press 'l' to create your first lens -4. Press 't' to generate a tour -""" - layout["content"].update(Panel(dashboard_content, title="Dashboard", border_style="green")) - - -def _render_scan_view(layout): - """Render the scan view.""" - # Sidebar - sidebar_content = """ -[bold]Scan Options[/bold] -• Target: Current directory -• Output: maps/out.json -• Verbose: False -• Debug: False - -[bold]Analysis Types[/bold] -• Python: AST analysis -• Dependencies: Import tracking -• Side effects: I/O detection -• Complexity: Cyclomatic metrics -""" - layout["sidebar"].update(Panel(sidebar_content, title="Scan Config", border_style="yellow")) - - # Main content - scan_content = """ -[bold yellow]Code Analysis Scanner[/bold yellow] - -This will analyze your codebase and generate: -• Function dependency graph -• Side effect analysis -• Complexity metrics -• Hot path identification - -[bold]Ready to scan?[/bold] -Press 'Enter' to start analysis or 'Esc' to go back. - -[dim]Note: Large codebases may take several minutes to analyze.[/dim] -""" - layout["content"].update(Panel(scan_content, title="Scan Code", border_style="yellow")) - - -def _render_lens_view(layout): - """Render the lens management view.""" - # Sidebar - sidebar_content = """ -[bold]Lens Types[/bold] -• Entry Points -• Hot Paths -• Side Effects -• High Complexity -• Custom Seeds - -[bold]Presets[/bold] -• bug: Error-related functions -• feature: New feature code -• api: API endpoints -• test: Test functions -""" - layout["sidebar"].update(Panel(sidebar_content, title="Lens Types", border_style="cyan")) - - # Main content - lens_content = """ -[bold cyan]Lens Management[/bold cyan] - -Lenses help you focus on specific parts of your codebase: -• Filter by function type -• Highlight important paths -• Isolate side effects -• Focus on complexity hotspots - -[bold]Create New Lens[/bold] -1. Choose lens type -2. Set seed functions -3. Configure filters -4. Save as preset - -[bold]Existing Lenses[/bold] -• No lenses created yet -""" - layout["content"].update(Panel(lens_content, title="Lens Management", border_style="cyan")) - - -def _render_tour_view(layout): - """Render the tour builder view.""" - # Sidebar - sidebar_content = """ -[bold]Tour Steps[/bold] -1. Introduction -2. Architecture Overview -3. Key Components -4. Data Flow -5. Side Effects -6. Testing Strategy - -[bold]Export Options[/bold] -• Markdown -• HTML -• PDF -• Interactive Web -""" - layout["sidebar"].update(Panel(sidebar_content, title="Tour Builder", border_style="magenta")) - - # Main content - tour_content = """ -[bold magenta]Tour Builder[/bold magenta] - -Generate interactive tours to help team members understand your codebase: -• Step-by-step walkthrough -• Code highlighting -• Interactive navigation -• Export for documentation - -[bold]Create New Tour[/bold] -1. Select starting functions -2. Define tour steps -3. Add explanations -4. Generate and export - -[bold]Tour Templates[/bold] -• Onboarding tour -• Architecture overview -• Feature walkthrough -• Debugging guide -""" - layout["content"].update(Panel(tour_content, title="Tour Builder", border_style="magenta")) - - -def _render_config_view(layout): - """Render the configuration view.""" - # Sidebar - sidebar_content = """ -[bold]Config Sections[/bold] -• Project Type -• Analysis Settings -• Seeds & Presets -• Metrics & Tracking -• CI Integration -• IDE Integration - -[bold]Current Status[/bold] -• Project: Not configured -• Analysis: Default settings -• Seeds: None defined -• Metrics: Disabled -""" - layout["sidebar"].update(Panel(sidebar_content, title="Configuration", border_style="red")) - - # Main content - config_content = """ -[bold red]Project Configuration[/bold red] - -Configure understand-first for your specific project: -• Set analysis depth and scope -• Define seed functions -• Configure metrics tracking -• Set up CI/CD integration - -[bold]Configuration Wizard[/bold] -Press 'w' to run the interactive wizard -or manually edit .understand-first.yml - -[bold]Current Settings[/bold] -• Hops: 2 -• Seeds: [] -• Metrics: Disabled -• CI: Not configured -""" - layout["content"].update(Panel(config_content, title="Configuration", border_style="red")) - - -def _show_help(layout): - """Show help information.""" - help_content = """ -[bold]Keyboard Shortcuts[/bold] - -Navigation: -• 1-5: Switch between views -• q: Quit application -• h: Show this help - -Actions: -• s: Start code scan -• l: Create new lens -• t: Generate tour -• c: Configure project -• w: Run configuration wizard - -General: -• Enter: Confirm action -• Esc: Go back -• Ctrl+C: Force quit -""" - layout["content"].update(Panel(help_content, title="Help", border_style="blue")) - - -def _run_config_wizard(): - """Run interactive configuration wizard with enhanced features.""" - print("[bold blue]🧠 Understand-First Configuration Wizard[/bold blue]") - print("This wizard will help you set up your .understand-first.yml configuration.") - print("The wizard will guide you through project-specific optimizations and best practices.") - print() - - # Enhanced project type selection - project_types = { - "1": ("python", "Python project", "General Python applications and libraries"), - "2": ( - "django", - "Django web application", - "Django web apps with models, views, and templates", - ), - "3": ("fastapi", "FastAPI web application", "Modern async API applications with FastAPI"), - "4": ("flask", "Flask web application", "Flask web apps with blueprints and extensions"), - "5": ( - "microservices", - "Microservices architecture", - "Distributed systems with multiple services", - ), - "6": ("react", "React frontend", "React applications with components and hooks"), - "7": ( - "nodejs", - "Node.js application", - "Node.js applications with Express or other frameworks", - ), - "8": ("go", "Go application", "Go applications and microservices"), - "9": ("java", "Java application", "Java applications with Spring or other frameworks"), - "10": ("custom", "Custom configuration", "Manually configure all settings"), - } - - print("What type of project are you configuring?") - for key, (value, description, details) in project_types.items(): - print(f" {key}. {description}") - print(f" {details}") - print() - - while True: - choice = typer.prompt("Enter your choice (1-10)", type=str) - if choice in project_types: - project_type, project_name, project_details = project_types[choice] - break - print("[red]Invalid choice. Please enter 1-10.[/red]") - - print(f"\n[green]Selected:[/green] {project_name}") - print(f"[dim]{project_details}[/dim]") - - # Load template if available - template_config = _load_project_template(project_type) - - # Configuration options with enhanced defaults - config = { - "hops": 2, - "seeds": [], - "seeds_for": {}, - "contracts_paths": [], - "glossary_path": "docs/glossary.md", - "metrics": {"enabled": False}, - "exclude_patterns": [], - "include_patterns": [], - "analysis_options": {}, - "ci_integration": {"enabled": False}, - "ide_integration": {"enabled": True}, - } - - # Merge template configuration - if template_config: - config.update(template_config) - print(f"\n[green]Loaded template configuration for {project_name}[/green]") - - # Hops configuration with better validation - print("\n[bold]Analysis Depth Configuration[/bold]") - print("Hops determine how deep the analysis should traverse from seed functions.") - print("Higher values provide more comprehensive analysis but may be slower.") - - hops = typer.prompt( - "How many hops should the lens traverse? (1-10, default: 2)", - type=int, - default=config.get("hops", 2), - ) - config["hops"] = max(1, min(10, hops)) # Clamp between 1 and 10 - - # Enhanced seeds configuration - print("\n[bold]Seeds Configuration[/bold]") - print("Seeds are starting points for understanding analysis.") - print("You can add files, functions, modules, or patterns as seeds.") - print("Examples: 'main.py', 'app.py:main', '*/models.py', 'service.*'") - - add_seeds = typer.confirm( - "Would you like to add custom seeds?", default=len(config.get("seeds", [])) == 0 - ) - if add_seeds: - seeds = config.get("seeds", []) - print("\n[dim]Enter seeds one by one. Press Enter with empty input to finish.[/dim]") - while True: - seed = typer.prompt( - "Enter a seed (file path, function, or pattern) or press Enter to finish", - default="", - ) - if not seed: - break - if seed not in seeds: - seeds.append(seed) - print(f"[green]Added seed:[/green] {seed}") - else: - print(f"[yellow]Seed already exists:[/yellow] {seed}") - config["seeds"] = seeds - - # Enhanced preset seeds configuration - print("\n[bold]Preset Seeds for Common Scenarios[/bold]") - print("Define preset seed collections for common development scenarios.") - print("These can be used with commands like 'u lens preset bug' or 'u lens preset feature'.") - - add_presets = typer.confirm( - "Would you like to configure preset seed collections?", default=True - ) - if add_presets: - presets = config.get("seeds_for", {}) - - # Suggest common presets based on project type - suggested_presets = _get_suggested_presets(project_type) - if suggested_presets: - print(f"\n[bold]Suggested presets for {project_name}:[/bold]") - for preset_name, preset_seeds in suggested_presets.items(): - print(f" • {preset_name}: {', '.join(preset_seeds)}") - - use_suggested = typer.confirm("Use these suggested presets?", default=True) - if use_suggested: - presets.update(suggested_presets) - - # Allow custom presets - add_custom_presets = typer.confirm("Add custom presets?", default=False) - if add_custom_presets: - while True: - preset_name = typer.prompt( - "Enter preset name (e.g., 'bug', 'feature', 'api') or press Enter to finish", - default="", - ) - if not preset_name: - break - - preset_seeds = [] - print(f"\n[dim]Enter seeds for '{preset_name}' preset:[/dim]") - while True: - seed = typer.prompt( - f"Enter seed for '{preset_name}' preset or press Enter to finish", - default="", - ) - if not seed: - break - preset_seeds.append(seed) - - if preset_seeds: - presets[preset_name] = preset_seeds - print( - f"[green]Added preset '{preset_name}' with {len(preset_seeds)} seeds[/green]" - ) - - config["seeds_for"] = presets - - # Enhanced contract configuration - print("\n[bold]Contract Configuration[/bold]") - print("Contracts define API specifications and formal verification requirements.") - print("These help ensure API compliance and can generate property tests.") - - add_contracts = typer.confirm("Do you have contract files to include?", default=False) - if add_contracts: - contract_paths = config.get("contracts_paths", []) - print("\n[dim]Enter contract file paths one by one:[/dim]") - while True: - path = typer.prompt("Enter contract file path or press Enter to finish", default="") - if not path: - break - if os.path.exists(path): - contract_paths.append(path) - print(f"[green]Added contract:[/green] {path}") - else: - print(f"[yellow]File not found:[/yellow] {path}") - add_anyway = typer.confirm("Add anyway?", default=False) - if add_anyway: - contract_paths.append(path) - config["contracts_paths"] = contract_paths - - # Enhanced metrics configuration - print("\n[bold]Metrics and Analytics Configuration[/bold]") - print("Understand-First can track Time To Understanding (TTU) metrics and generate reports.") - - enable_metrics = typer.confirm("Enable TTU metrics tracking?", default=False) - if enable_metrics: - config["metrics"]["enabled"] = True - - # Additional metrics options - track_weekly = typer.confirm("Generate weekly TTU reports?", default=True) - if track_weekly: - config["metrics"]["weekly_reports"] = True - - track_events = typer.confirm( - "Track specific events (onboarding, debugging, etc.)?", default=True - ) - if track_events: - config["metrics"]["event_tracking"] = True - else: - config["metrics"]["enabled"] = False - - # CI/CD Integration - print("\n[bold]CI/CD Integration[/bold]") - print("Configure Understand-First to run in your CI/CD pipeline.") - - enable_ci = typer.confirm("Enable CI integration?", default=False) - if enable_ci: - config["ci_integration"]["enabled"] = True - - ci_platform = typer.prompt( - "CI platform (github, gitlab, jenkins, other)", default="github" - ).lower() - config["ci_integration"]["platform"] = ci_platform - - fail_on_issues = typer.confirm("Fail CI on understanding issues?", default=True) - config["ci_integration"]["fail_on_issues"] = fail_on_issues - - generate_reports = typer.confirm("Generate CI reports?", default=True) - config["ci_integration"]["generate_reports"] = generate_reports - - # IDE Integration - print("\n[bold]IDE Integration[/bold]") - print("Configure IDE-specific features and integrations.") - - enable_ide = typer.confirm("Enable IDE integration?", default=True) - if enable_ide: - config["ide_integration"]["enabled"] = True - - ide_type = typer.prompt("IDE type (vscode, pycharm, vim, other)", default="vscode").lower() - config["ide_integration"]["type"] = ide_type - - show_gutter_annotations = typer.confirm("Show gutter annotations?", default=True) - config["ide_integration"]["gutter_annotations"] = show_gutter_annotations - - enable_quick_peek = typer.confirm("Enable quick peek tours?", default=True) - config["ide_integration"]["quick_peek"] = enable_quick_peek - - # Analysis options - print("\n[bold]Analysis Options[/bold]") - print("Configure advanced analysis features and optimizations.") - - analysis_options = {} - - enable_complexity_analysis = typer.confirm("Enable complexity analysis?", default=True) - if enable_complexity_analysis: - analysis_options["complexity_analysis"] = True - - enable_side_effect_detection = typer.confirm("Enable side effect detection?", default=True) - if enable_side_effect_detection: - analysis_options["side_effects"] = True - - enable_dependency_analysis = typer.confirm("Enable dependency analysis?", default=True) - if enable_dependency_analysis: - analysis_options["dependencies"] = True - - config["analysis_options"] = analysis_options - - # File patterns - print("\n[bold]File Inclusion/Exclusion Patterns[/bold]") - print("Configure which files to include or exclude from analysis.") - - configure_patterns = typer.confirm("Configure file patterns?", default=False) - if configure_patterns: - # Include patterns - include_patterns = [] - print("\n[dim]Enter include patterns (e.g., '*.py', 'src/**/*.js'):[/dim]") - while True: - pattern = typer.prompt("Include pattern or press Enter to finish", default="") - if not pattern: - break - include_patterns.append(pattern) - config["include_patterns"] = include_patterns - - # Exclude patterns - exclude_patterns = ["**/__pycache__/**", "**/node_modules/**", "**/.git/**"] - print(f"\n[dim]Default exclude patterns: {', '.join(exclude_patterns)}[/dim]") - add_excludes = typer.confirm("Add additional exclude patterns?", default=False) - if add_excludes: - while True: - pattern = typer.prompt("Exclude pattern or press Enter to finish", default="") - if not pattern: - break - exclude_patterns.append(pattern) - config["exclude_patterns"] = exclude_patterns - - # Write configuration with validation - try: - import yaml - - # Validate configuration - errors = validate_config_dict(config) - if errors: - print("\n[red]Configuration validation errors:[/red]") - for error in errors: - print(f" • {error}") - - fix_errors = typer.confirm("Fix errors automatically?", default=True) - if fix_errors: - config = _fix_config_errors(config, errors) - else: - print("[yellow]Saving configuration with errors. Please fix manually.[/yellow]") - - config_yaml = yaml.dump(config, default_flow_style=False, sort_keys=False, indent=2) - - with open(".understand-first.yml", "w", encoding="utf-8") as f: - f.write(config_yaml) - - # Create necessary directories - directories = ["tours", "maps", "traces", "contracts", "docs", "fixtures"] - for directory in directories: - os.makedirs(directory, exist_ok=True) - - # Create example files - _create_example_files(project_type) - - # Update README if it exists - _update_readme_with_integration(project_type) - - print("\n[green]✅ Configuration saved to .understand-first.yml[/green]") - print("\n[bold]Next steps:[/bold]") - print("1. Run `u scan . -o maps/repo.json` to generate a repository map") - print("2. Run `u demo` for a guided demonstration") - print("3. Run `u doctor` to verify your setup") - print("4. Run `u init --template` to use project templates") - - if config.get("ci_integration", {}).get("enabled"): - print("5. Configure your CI pipeline to run `u ci`") - - print("\n[dim]Configuration wizard completed successfully![/dim]") - - except Exception as e: - print(f"\n[red]Error saving configuration:[/red] {e}") - raise typer.Exit(1) - - -def _load_project_template(project_type: str) -> Optional[Dict[str, Any]]: - """Load project template configuration if available.""" - template_path = f"templates/{project_type}/.understand-first.yml" - - if os.path.exists(template_path): - try: - import yaml - - with open(template_path, "r", encoding="utf-8") as f: - return yaml.safe_load(f) - except Exception as e: - print(f"Warning: Failed to load template {template_path}: {e}") - - return None - - -def _get_suggested_presets(project_type: str) -> Dict[str, List[str]]: - """Get suggested presets based on project type.""" - presets = { - "python": { - "main": ["main.py", "app.py", "run.py"], - "tests": ["test_*.py", "tests/*.py"], - "utils": ["utils/*.py", "helpers/*.py"], - }, - "django": { - "models": ["*/models.py"], - "views": ["*/views.py"], - "urls": ["*/urls.py"], - "admin": ["*/admin.py"], - "forms": ["*/forms.py"], - "tests": ["*/test*.py", "*/tests.py"], - }, - "fastapi": { - "routes": ["*/routes/*.py", "*/api/*.py", "main.py"], - "models": ["*/models.py", "*/schemas.py"], - "services": ["*/services/*.py", "*/core/*.py"], - "tests": ["test_*.py", "*/test_*.py"], - }, - "flask": { - "routes": ["app.py", "*/routes/*.py", "*/blueprints/*.py"], - "models": ["*/models.py"], - "forms": ["*/forms.py"], - "templates": ["*/templates/**/*.html"], - "tests": ["test_*.py", "*/test_*.py"], - }, - "microservices": { - "services": ["*/service*.py", "*/api*.py"], - "clients": ["*/client*.py"], - "models": ["*/models.py", "*/schemas.py"], - "tests": ["*/test*.py", "*/tests.py"], - }, - "react": { - "components": ["src/components/**/*.js", "src/components/**/*.jsx"], - "pages": ["src/pages/**/*.js", "src/pages/**/*.jsx"], - "hooks": ["src/hooks/**/*.js"], - "utils": ["src/utils/**/*.js", "src/helpers/**/*.js"], - "tests": ["src/**/*.test.js", "src/**/*.spec.js"], - }, - "nodejs": { - "routes": ["routes/*.js", "*/routes/*.js"], - "controllers": ["controllers/*.js", "*/controllers/*.js"], - "models": ["models/*.js", "*/models/*.js"], - "middleware": ["middleware/*.js", "*/middleware/*.js"], - "tests": ["test/*.js", "*/test/*.js", "**/*.test.js"], - }, - "go": { - "main": ["main.go", "cmd/**/*.go"], - "handlers": ["handlers/*.go", "*/handlers/*.go"], - "models": ["models/*.go", "*/models/*.go"], - "services": ["services/*.go", "*/services/*.go"], - "tests": ["*_test.go", "**/*_test.go"], - }, - "java": { - "controllers": ["**/controller/*.java", "**/web/*.java"], - "services": ["**/service/*.java", "**/business/*.java"], - "models": ["**/model/*.java", "**/entity/*.java"], - "repositories": ["**/repository/*.java", "**/dao/*.java"], - "tests": ["**/test/**/*.java", "**/*Test.java"], - }, - } - - return presets.get(project_type, {}) - - -def _fix_config_errors(config: Dict[str, Any], errors: List[str]) -> Dict[str, Any]: - """Fix common configuration errors automatically.""" - fixed_config = config.copy() - - for error in errors: - if "hops" in error and "must be between" in error: - fixed_config["hops"] = max(1, min(10, fixed_config.get("hops", 2))) - elif "seeds_for" in error and "must be a dict" in error: - fixed_config["seeds_for"] = {} - elif "metrics" in error and "must be a dict" in error: - fixed_config["metrics"] = {"enabled": False} - elif "contracts_paths" in error and "must be a list" in error: - fixed_config["contracts_paths"] = [] - elif "exclude_patterns" in error and "must be a list" in error: - fixed_config["exclude_patterns"] = ["**/__pycache__/**", "**/node_modules/**"] - elif "include_patterns" in error and "must be a list" in error: - fixed_config["include_patterns"] = [] - - return fixed_config - - -def _create_example_files(project_type: str) -> None: - """Create example files based on project type.""" - try: - # Create example tour - tour_content = f"""# Understanding Tour Example - -This is an example tour generated for a {project_type} project. - -## Getting Started - -1. Run `u scan . -o maps/repo.json` to generate a repository map -2. Run `u lens from-seeds --map maps/repo.json --seed main -o maps/lens.json` -3. Run `u tour maps/lens.json -o tours/understanding.md` - -## Project-Specific Tips - -For {project_type} projects, focus on: -- Main entry points and application structure -- Key business logic and data models -- API endpoints and routing -- Test coverage and quality - -## Next Steps - -- Add more seeds based on your specific use cases -- Configure CI integration for automated analysis -- Set up IDE integration for real-time insights -""" - - with open("tours/example.md", "w", encoding="utf-8") as f: - f.write(tour_content) - - # Create example fixture - fixture_content = f"""# Example Fixture for {project_type} Project - -This fixture demonstrates how to test the understanding generated by Understand-First. - -## Usage - -Run this fixture to verify that the understanding tour can be executed: - -```bash -u tour_run --fixtures fixtures maps/lens.json -``` - -## Customization - -Modify this fixture to match your project's specific requirements and test scenarios. -""" - - with open("fixtures/example_fixture.py", "w", encoding="utf-8") as f: - f.write(fixture_content) - - # Create example contract - contract_content = f"""# Example Contract for {project_type} Project - -This is an example contract file that defines API specifications and formal verification requirements. - -## Contract Definition - -```yaml -# Example API contract -ROUTE::api: - GET /health: - request_schema: {{}} - response_schema: - type: object - properties: - status: - type: string - enum: [healthy, unhealthy] - timestamp: - type: string - format: date-time - preconditions: [] - postconditions: ["response.status_code == 200"] - side_effects: [] -``` - -## Usage - -1. Define your API contracts in this file -2. Run `u contracts verify` to check compliance -3. Generate property tests with `u contracts stub-tests` -""" - - with open("contracts/example_contracts.yaml", "w", encoding="utf-8") as f: - f.write(contract_content) - - print("[green]Created example files:[/green]") - print(" • tours/example.md") - print(" • fixtures/example_fixture.py") - print(" • contracts/example_contracts.yaml") - - except Exception as e: - print(f"Warning: Failed to create example files: {e}") - - -def _update_readme_with_integration(project_type: str) -> None: - """Update README with Understand-First integration section.""" - if not os.path.exists("README.md"): - return - - try: - integration_section = f""" - -## Understand-First Integration - -This project uses [Understand-First](https://github.com/your-org/understand-first) for automated code understanding and documentation generation. - -### Quick Start - -1. **Generate Repository Map** - ```bash - u scan . -o maps/repo.json - ``` - -2. **Create Understanding Lens** - ```bash - u lens from-seeds --map maps/repo.json --seed main -o maps/lens.json - ``` - -3. **Generate Understanding Tour** - ```bash - u tour maps/lens.json -o tours/understanding.md - ``` - -4. **Run Guided Demo** - ```bash - u demo - ``` - -### Project-Specific Configuration - -This {project_type} project is configured with optimized settings for: -- **Seeds**: Key entry points and important modules -- **Analysis**: Complexity analysis, side effect detection, and dependency tracking -- **Integration**: IDE support and CI/CD pipeline integration - -### Understanding Features - -- **Interactive Tours**: Step-by-step walkthroughs of complex code paths -- **Runtime Tracing**: Actual execution paths, not just static analysis -- **Contract Verification**: API compliance and formal verification -- **Metrics Tracking**: Time To Understanding (TTU) measurement -- **IDE Integration**: Real-time insights in your development environment - -### Configuration - -The project configuration is stored in `.understand-first.yml`. Key settings include: - -- **Hops**: Analysis depth (currently set to {2}) -- **Seeds**: Starting points for analysis -- **Presets**: Common scenarios like bug fixes and feature development -- **Patterns**: File inclusion/exclusion rules - -### CI/CD Integration - -The project is configured for CI/CD integration. Add this to your pipeline: - -```yaml -- name: Understand-First Analysis - run: | - u scan . -o maps/repo.json - u lens preset feature --map maps/repo.json -o maps/lens.json - u tour maps/lens.json -o tours/ci-tour.md -``` - -### IDE Integration - -Install the Understand-First VS Code extension for: -- Gutter annotations showing complexity and call counts -- Quick peek tours and explanations -- Real-time understanding insights - -### Learn More - -- [Documentation](https://github.com/your-org/understand-first#readme) -- [Examples](https://github.com/your-org/understand-first/tree/main/examples) -- [Web Demo](https://your-org.github.io/understand-first/demo) -""" - - with open("README.md", "a", encoding="utf-8") as f: - f.write(integration_section) - - print("[green]Updated README.md with Understand-First integration section[/green]") - - except Exception as e: - print(f"Warning: Failed to update README: {e}") - - -def generate_diff_markdown(added_functions, removed_functions, modified_functions, changes_count): - """Generate Markdown summary for diff analysis.""" - markdown = f"""# Understanding Delta Analysis - -## Summary - -- **Total Changes**: {changes_count} -- **Functions Added**: {len(added_functions)} -- **Functions Removed**: {len(removed_functions)} -- **Functions Modified**: {len(modified_functions)} - -## Added Functions - -""" - - if added_functions: - for func in sorted(added_functions): - markdown += f"- `{func}`\n" - else: - markdown += "No functions added.\n" - - markdown += "\n## Removed Functions\n\n" - - if removed_functions: - for func in sorted(removed_functions): - markdown += f"- `{func}`\n" - else: - markdown += "No functions removed.\n" - - markdown += "\n## Modified Functions\n\n" - - if modified_functions: - for func in sorted(modified_functions): - markdown += f"- `{func}`\n" - else: - markdown += "No functions modified.\n" - - markdown += """ -## Review Checklist - -- [ ] Review added functions for new side effects -- [ ] Verify removed functions don't break dependencies -- [ ] Check modified functions for complexity changes -- [ ] Update understanding tours if needed -- [ ] Update documentation for significant changes - -*Generated by Understand-First* -""" - - return markdown - - -def generate_enhanced_diff_markdown( - added_functions, - removed_functions, - modified_functions, - policy_breaches, - changes_count, - policy_threshold, -): - """Generate enhanced Markdown summary of diff analysis with policy breach information.""" - content = f"# Delta Analysis Summary\n\n" - content += f"**Total Changes:** {changes_count}\n" - content += f"**Policy Breaches:** {len(policy_breaches)}\n" - content += f"**Policy Threshold:** {policy_threshold}\n\n" - - # Status indicator - if policy_breaches: - content += "## 🚨 Policy Breaches Detected\n\n" - content += ( - f"The following functions exceed the complexity threshold of {policy_threshold}:\n\n" - ) - for breach in policy_breaches: - content += f"- **{breach['function']}**: {breach['old_complexity']} → {breach['new_complexity']} (threshold: {breach['threshold']})\n" - content += "\n" - elif changes_count > 0: - content += "## ⚠️ Changes Detected\n\n" - content += "The following changes were detected in your codebase:\n\n" - else: - content += "## ✅ No Changes Detected\n\n" - content += "Your codebase has no changes since the last analysis.\n\n" - - if added_functions: - content += f"## Added Functions ({len(added_functions)})\n\n" - for func in sorted(added_functions): - content += f"- `{func}`\n" - content += "\n" - - if removed_functions: - content += f"## Removed Functions ({len(removed_functions)})\n\n" - for func in sorted(removed_functions): - content += f"- `{func}`\n" - content += "\n" - - if modified_functions: - content += f"## Modified Functions ({len(modified_functions)})\n\n" - for func in sorted(modified_functions): - content += f"- `{func}`\n" - content += "\n" - - content += "## Review Checklist\n\n" - content += "- [ ] Review added functions for new side effects\n" - content += "- [ ] Verify removed functions don't break dependencies\n" - content += "- [ ] Check modified functions for complexity changes\n" - if policy_breaches: - content += f"- [ ] **Address policy breaches (complexity > {policy_threshold})**\n" - content += "- [ ] Update documentation if needed\n" - content += "- [ ] Update understanding tours if needed\n" - content += "- [ ] Update CI/CD policies if needed\n\n" - - content += "## Exit Codes\n\n" - content += "The following exit codes are used for CI integration:\n\n" - content += "- **0**: No changes detected\n" - content += "- **2**: Changes detected (update tours/documentation)\n" - content += "- **3**: Policy breaches detected (address complexity issues)\n\n" - - content += "*Generated by Understand-First*" - - return content - - -@app.command() -def tour_gate(progress_json: str = typer.Option(".uf-progress.json", "--progress")): - """Fail if walkthrough milestones are not met (Opened 3/3, Ran 3/3).""" - try: - data = json.load(open(progress_json, "r", encoding="utf-8")) - except Exception: - print("[red]No progress file found[/red]") - raise typer.Exit(1) - opened = int(data.get("opened", 0)) - ran = int(data.get("ran", 0)) - if opened >= 3 and ran >= 3: - print("[green]Tour milestones met[/green]") - return - print(f"[red]Tour milestones not met[/red]: opened {opened}/3 ran {ran}/3") - raise typer.Exit(1) - - -@app.command() -def diff( - old_lens: str = typer.Option(..., "--old"), - new_lens: str = typer.Option(..., "--new"), - o: str = typer.Option("maps/delta.svg", "--output", "-o"), - verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable verbose output"), - json_output: bool = typer.Option(False, "--json", help="Output JSON delta instead of SVG"), - markdown: bool = typer.Option(False, "--markdown", help="Output Markdown summary"), - ci_gate: bool = typer.Option(False, "--ci-gate", help="Exit with non-zero code for CI gates"), - policy_threshold: int = typer.Option( - 5, "--policy-threshold", help="Complexity threshold for policy breach" - ), -): - """Compare two lens files and generate delta visualization with enhanced analysis.""" - console = Console() - - try: - # Enhanced validation with better error messages - if not pathlib.Path(old_lens).exists(): - console.print( - Panel.fit( - f"[red]Error: Old lens file '{old_lens}' does not exist[/red]\n\n" - "This could mean:\n" - "• The file path is incorrect\n" - "• The file hasn't been generated yet\n" - "• There's a typo in the filename", - title="File Not Found", - border_style="red", - ) - ) - raise typer.Exit(1) - - if not pathlib.Path(new_lens).exists(): - console.print( - Panel.fit( - f"[red]Error: New lens file '{new_lens}' does not exist[/red]\n\n" - "This could mean:\n" - "• The file path is incorrect\n" - "• The file hasn't been generated yet\n" - "• There's a typo in the filename", - title="File Not Found", - border_style="red", - ) - ) - raise typer.Exit(1) - - # Load lens data with error handling - try: - with open(old_lens, "r", encoding="utf-8") as f: - old_data = json.load(f) - except json.JSONDecodeError as e: - console.print(f"[red]Error parsing old lens file:[/red] {str(e)}") - raise typer.Exit(1) - - try: - with open(new_lens, "r", encoding="utf-8") as f: - new_data = json.load(f) - except json.JSONDecodeError as e: - console.print(f"[red]Error parsing new lens file:[/red] {str(e)}") - raise typer.Exit(1) - - # Enhanced configuration display - config_table = Table(title="Delta Analysis Configuration", show_header=False) - config_table.add_column("Setting", style="cyan") - config_table.add_column("Value", style="white") - config_table.add_row("Old Lens", old_lens) - config_table.add_row("New Lens", new_lens) - config_table.add_row("Output Format", "JSON" if json_output else "SVG") - config_table.add_row("Output File", o) - config_table.add_row("Policy Threshold", str(policy_threshold)) - config_table.add_row("CI Gate", "Enabled" if ci_gate else "Disabled") - - console.print(config_table) - console.print() - - # Enhanced analysis with better progress tracking - with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - BarColumn(), - TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), - TimeElapsedColumn(), - console=console, - ) as progress: - - diff_task = progress.add_task("Analyzing differences...", total=100) - progress.update(diff_task, advance=10) - - # Calculate statistics - old_functions = set(old_data.get("functions", {}).keys()) - new_functions = set(new_data.get("functions", {}).keys()) - - added_functions = new_functions - old_functions - removed_functions = old_functions - new_functions - modified_functions = [] - policy_breaches = [] - - progress.update(diff_task, advance=20) - - # Enhanced function comparison - for func in old_functions & new_functions: - old_func = old_data["functions"][func] - new_func = new_data["functions"][func] - - # Check for modifications - if ( - old_func.get("complexity", 0) != new_func.get("complexity", 0) - or old_func.get("side_effects", []) != new_func.get("side_effects", []) - or old_func.get("lines", 0) != new_func.get("lines", 0) - ): - modified_functions.append(func) - - # Check for policy breaches - if new_func.get("complexity", 0) > policy_threshold: - policy_breaches.append( - { - "function": func, - "old_complexity": old_func.get("complexity", 0), - "new_complexity": new_func.get("complexity", 0), - "threshold": policy_threshold, - } - ) - - progress.update(diff_task, advance=30) - - # Check added functions for policy breaches - for func in added_functions: - func_data = new_data["functions"][func] - if func_data.get("complexity", 0) > policy_threshold: - policy_breaches.append( - { - "function": func, - "old_complexity": 0, - "new_complexity": func_data.get("complexity", 0), - "threshold": policy_threshold, - } - ) - - progress.update(diff_task, advance=20) - - # Generate enhanced output - if json_output: - delta_data = { - "summary": { - "added": len(added_functions), - "removed": len(removed_functions), - "modified": len(modified_functions), - "total_changes": len(added_functions) - + len(removed_functions) - + len(modified_functions), - "policy_breaches": len(policy_breaches), - "policy_threshold": policy_threshold, - }, - "added_functions": list(added_functions), - "removed_functions": list(removed_functions), - "modified_functions": modified_functions, - "policy_breaches": policy_breaches, - "old_lens": old_lens, - "new_lens": new_lens, - "timestamp": datetime.now().isoformat(), - } - - os.makedirs(pathlib.Path(o).parent, exist_ok=True) - with open(o, "w", encoding="utf-8") as f: - json.dump(delta_data, f, indent=2) - else: - # Generate SVG visualization - svg = lens_delta_svg(old_lens, new_lens) - with open(o, "w", encoding="utf-8") as df: - df.write(svg) - - progress.update(diff_task, advance=20) - - # Enhanced results display - changes_count = len(added_functions) + len(removed_functions) + len(modified_functions) - - # Determine border style based on policy breaches - if policy_breaches: - border_style = "red" - status_icon = "🚨" - status_text = "Policy breaches detected!" - elif changes_count > 0: - border_style = "yellow" - status_icon = "⚠️" - status_text = "Changes detected" - else: - border_style = "green" - status_icon = "✅" - status_text = "No changes detected" - - results_panel = Panel( - f"[green]✓ Delta analysis completed![/green]\n\n" - f"📊 [bold]Changes Summary:[/bold]\n" - f" • Functions added: {len(added_functions)}\n" - f" • Functions removed: {len(removed_functions)}\n" - f" • Functions modified: {len(modified_functions)}\n" - f" • Total changes: {changes_count}\n" - f" • Policy breaches: {len(policy_breaches)}\n" - f" • Output file: {o}\n\n" - f"🎯 [bold]Review Impact:[/bold]\n" - f" • Check added functions for new side effects\n" - f" • Verify removed functions don't break dependencies\n" - f" • Review modified functions for complexity changes\n" - f" • Address policy breaches (complexity > {policy_threshold})", - title=f"{status_icon} Delta Analysis Results - {status_text}", - border_style=border_style, - ) - - console.print(results_panel) - - # Show policy breaches if any - if policy_breaches: - console.print("\n[bold red]Policy Breaches Detected:[/bold red]") - breach_table = Table(title="Functions Exceeding Complexity Threshold") - breach_table.add_column("Function", style="cyan") - breach_table.add_column("Old Complexity", justify="center", style="yellow") - breach_table.add_column("New Complexity", justify="center", style="red") - breach_table.add_column("Threshold", justify="center", style="magenta") - - for breach in policy_breaches: - breach_table.add_row( - breach["function"], - str(breach["old_complexity"]), - str(breach["new_complexity"]), - str(breach["threshold"]), - ) - - console.print(breach_table) - - # Show detailed changes if verbose - if verbose and changes_count > 0: - if added_functions: - console.print("\n[bold green]Added Functions:[/bold green]") - for func in sorted(added_functions): - func_data = new_data["functions"][func] - complexity = func_data.get("complexity", 0) - side_effects = func_data.get("side_effects", []) - effects_str = ( - f" (side effects: {', '.join(side_effects)})" if side_effects else "" - ) - console.print(f" + {func} (complexity: {complexity}){effects_str}") - - if removed_functions: - console.print("\n[bold red]Removed Functions:[/bold red]") - for func in sorted(removed_functions): - console.print(f" - {func}") - - if modified_functions: - console.print("\n[bold yellow]Modified Functions:[/bold yellow]") - for func in sorted(modified_functions): - old_func = old_data["functions"][func] - new_func = new_data["functions"][func] - old_complexity = old_func.get("complexity", 0) - new_complexity = new_func.get("complexity", 0) - console.print(f" ~ {func} (complexity: {old_complexity} → {new_complexity})") - - # Track TTU metric - ttu_record( - "diff_analyzed", - { - "changes_count": changes_count, - "added": len(added_functions), - "removed": len(removed_functions), - "modified": len(modified_functions), - "policy_breaches": len(policy_breaches), - }, - ) - - # Generate enhanced Markdown summary if requested - if markdown: - markdown_content = generate_enhanced_diff_markdown( - added_functions, - removed_functions, - modified_functions, - policy_breaches, - changes_count, - policy_threshold, - ) - markdown_path = o.replace(".svg", ".md").replace(".json", ".md") - with open(markdown_path, "w", encoding="utf-8") as f: - f.write(markdown_content) - console.print(f"[green]✓ Markdown summary written to {markdown_path}[/green]") - - # Enhanced exit codes for CI - if ci_gate: - if policy_breaches: - console.print( - f"\n[red]🚨 Policy breaches detected! {len(policy_breaches)} functions exceed complexity threshold of {policy_threshold}.[/red]" - ) - raise typer.Exit(3) # Policy breach exit code - elif changes_count > 0: - console.print( - "\n[yellow]⚠ Changes detected. Consider updating tours and documentation.[/yellow]" - ) - raise typer.Exit(2) # Changes detected exit code - else: - console.print("\n[green]✓ No changes detected.[/green]") - raise typer.Exit(0) # No changes exit code - else: - if policy_breaches: - console.print( - f"\n[red]🚨 Policy breaches detected! {len(policy_breaches)} functions exceed complexity threshold of {policy_threshold}.[/red]" - ) - elif changes_count > 0: - console.print( - "\n[yellow]⚠ Changes detected. Consider updating tours and documentation.[/yellow]" - ) - else: - console.print("\n[green]✓ No changes detected.[/green]") - raise typer.Exit(0) - - except Exception as e: - console.print( - Panel.fit( - f"[red]Error during diff analysis:[/red] {str(e)}\n\n" - "This might be due to:\n" - "• Corrupted lens files\n" - "• File permission issues\n" - "• Invalid JSON format\n\n" - "Try running with --verbose for more details", - title="Analysis Error", - border_style="red", - ) - ) - if verbose: - import traceback - - console.print(f"[dim]{traceback.format_exc()}[/dim]") - raise typer.Exit(1) - - -@app.command() -def metrics( - dashboard: bool = typer.Option(False, "--dashboard", "-d", help="Show metrics dashboard"), - days: int = typer.Option(30, "--days", help="Number of days to analyze"), - export: str = typer.Option(None, "--export", "-e", help="Export metrics to file"), - track: str = typer.Option(None, "--track", "-t", help="Track a specific event"), -): - """View and manage Understand-First metrics for TTU/TTFSC goals.""" - console = Console() - - try: - if track: - # Track a specific event - session_id = get_tracker().start_session() - track_event(track, session_id) - console.print(f"[green]✓ Tracked event: {track}[/green]") - return - - # Generate dashboard data - dashboard_data = get_dashboard_data(days) - - if dashboard: - # Show comprehensive dashboard - show_metrics_dashboard(console, dashboard_data) - else: - # Show summary metrics - show_metrics_summary(console, dashboard_data) - - # Export if requested - if export: - with open(export, "w", encoding="utf-8") as f: - json.dump(dashboard_data, f, indent=2) - console.print(f"[green]✓ Metrics exported to {export}[/green]") - - except Exception as e: - console.print(f"[red]Error with metrics:[/red] {str(e)}") - raise typer.Exit(1) - - -def show_metrics_dashboard(console: Console, data: Dict[str, Any]): - """Display comprehensive metrics dashboard""" - - # Header - console.print( - Panel( - "[bold blue]📊 Understand-First Metrics Dashboard[/bold blue]\n" - f"Period: Last {data['period_days']} days\n" - f"Generated: {datetime.fromtimestamp(data['timestamp']).strftime('%Y-%m-%d %H:%M:%S')}", - title="Dashboard", - border_style="blue", - ) - ) - - # North Star Goals Status - goals_table = Table(title="North Star Goals Status", show_header=True) - goals_table.add_column("Goal", style="cyan") - goals_table.add_column("Target", style="yellow") - goals_table.add_column("Current", style="white") - goals_table.add_column("Status", style="green") - - # TTU Goal - ttu_metrics = data["ttu_metrics"] - ttu_current = ttu_metrics["average_ttu_minutes"] - ttu_target = data["north_star_goals"]["ttu_target"] - ttu_status = "✅" if ttu_current and ttu_current <= ttu_target else "❌" - goals_table.add_row( - "TTU (minutes)", - f"≤{ttu_target}", - f"{ttu_current:.1f}" if ttu_current else "N/A", - ttu_status, - ) - - # Activation Goal - activation_metrics = data["activation_metrics"] - activation_current = activation_metrics["activation_rate_percentage"] - activation_target = data["north_star_goals"]["activation_target"] - activation_status = "✅" if activation_current >= activation_target else "❌" - goals_table.add_row( - "Activation Rate (%)", - f"≥{activation_target}", - f"{activation_current:.1f}", - activation_status, - ) - - # Tour Completion Goal - tour_metrics = data["tour_completion_metrics"] - tour_current = tour_metrics["completion_rate_percentage"] - tour_target = data["north_star_goals"]["tour_completion_target"] - tour_status = "✅" if tour_current >= tour_target else "❌" - goals_table.add_row( - "Tour Completion (%)", f"≥{tour_target}", f"{tour_current:.1f}", tour_status - ) - - # PR Coverage Goal - pr_metrics = data["pr_coverage_metrics"] - pr_current = pr_metrics["coverage_rate_percentage"] - pr_target = data["north_star_goals"]["pr_coverage_target"] - pr_status = "✅" if pr_current >= pr_target else "❌" - goals_table.add_row("PR Coverage (%)", f"≥{pr_target}", f"{pr_current:.1f}", pr_status) - - console.print(goals_table) - console.print() - - # Detailed Metrics - # TTU Metrics - ttu_panel = Panel( - f"📈 **TTU Metrics**\n" - f"• Total sessions: {ttu_metrics['total_sessions']}\n" - f"• Sessions with tour completion: {ttu_metrics['sessions_with_tour_completion']}\n" - f"• Average TTU: {ttu_metrics['average_ttu_minutes']:.1f} minutes\n" - f"• Under 10 minutes: {ttu_metrics['ttu_under_10_min_percentage']:.1f}%", - title="Time-to-Understanding", - border_style="blue", - ) - console.print(ttu_panel) - - # Activation Metrics - activation_panel = Panel( - f"🚀 **Activation Metrics**\n" - f"• Total sessions: {activation_metrics['total_sessions']}\n" - f"• Activated sessions: {activation_metrics['activated_sessions']}\n" - f"• Activation rate: {activation_metrics['activation_rate_percentage']:.1f}%\n" - f"• Under 2 minutes: {activation_metrics['under_2_min_percentage']:.1f}%", - title="User Activation", - border_style="green", - ) - console.print(activation_panel) - - # Tour Completion Metrics - tour_panel = Panel( - f"📚 **Tour Completion Metrics**\n" - f"• Total tours: {tour_metrics['total_tours']}\n" - f"• Completed tours: {tour_metrics['completed_tours']}\n" - f"• Completion rate: {tour_metrics['completion_rate_percentage']:.1f}%\n" - f"• Average completion: {tour_metrics['average_completion_percentage']:.1f}%", - title="Tour Completion", - border_style="yellow", - ) - console.print(tour_panel) - - # PR Coverage Metrics - pr_panel = Panel( - f"🔀 **PR Coverage Metrics**\n" - f"• Total PRs: {pr_metrics['total_prs']}\n" - f"• PRs with artifacts: {pr_metrics['prs_with_artifacts']}\n" - f"• Coverage rate: {pr_metrics['coverage_rate_percentage']:.1f}%", - title="PR Coverage", - border_style="red", - ) - console.print(pr_panel) - - -def show_metrics_summary(console: Console, data: Dict[str, Any]): - """Display metrics summary""" - - # Quick summary table - summary_table = Table( - title=f"Metrics Summary (Last {data['period_days']} days)", show_header=True - ) - summary_table.add_column("Metric", style="cyan") - summary_table.add_column("Value", style="white") - summary_table.add_column("Target", style="yellow") - - # Add key metrics - ttu_metrics = data["ttu_metrics"] - activation_metrics = data["activation_metrics"] - tour_metrics = data["tour_completion_metrics"] - pr_metrics = data["pr_coverage_metrics"] - - summary_table.add_row( - "Average TTU (minutes)", - ( - f"{ttu_metrics['average_ttu_minutes']:.1f}" - if ttu_metrics["average_ttu_minutes"] - else "N/A" - ), - "≤10", - ) - - summary_table.add_row( - "Activation Rate (%)", f"{activation_metrics['activation_rate_percentage']:.1f}", "≥80" - ) - - summary_table.add_row( - "Tour Completion (%)", f"{tour_metrics['completion_rate_percentage']:.1f}", "≥80" - ) - - summary_table.add_row("PR Coverage (%)", f"{pr_metrics['coverage_rate_percentage']:.1f}", "≥90") - - console.print(summary_table) - console.print() - - # Recommendations - recommendations = [] - - if not ttu_metrics["average_ttu_minutes"] or ttu_metrics["average_ttu_minutes"] > 10: - recommendations.append( - "🎯 Focus on reducing TTU: improve onboarding and tutorial experience" - ) - - if activation_metrics["activation_rate_percentage"] < 80: - recommendations.append( - "🚀 Improve activation: make map generation faster and more intuitive" - ) - - if tour_metrics["completion_rate_percentage"] < 80: - recommendations.append("📚 Enhance tour experience: make tours more engaging and shorter") - - if pr_metrics["coverage_rate_percentage"] < 90: - recommendations.append( - "🔀 Increase PR coverage: automate understanding artifact generation" - ) - - if recommendations: - console.print( - Panel("\n".join(recommendations), title="Recommendations", border_style="yellow") - ) - - -@app.command() -def config_validate(path: str = typer.Option(".understand-first.yml", "--path")): - if not os.path.exists(path): - print(f"[yellow]No config found at {path}[/yellow]") - raise typer.Exit(code=1) - import yaml as _yaml - - try: - data = _yaml.safe_load(open(path, "r", encoding="utf-8")) or {} - except Exception as e: - print(f"[red]YAML error:[/red] {e}") - raise typer.Exit(code=1) - errors = validate_config_dict(data) - if errors: - print("[red]Config errors:[/red]") - for e in errors: - print(f"- {e}") - raise typer.Exit(code=1) - print("[green]Config OK[/green]") - - -@lens_app.command("explain") -def lens_explain( - qname: str, - lens: str = typer.Option("maps/lens_merged.json", "--lens"), - repo: str = typer.Option("maps/repo.json", "--repo"), - json_out: bool = typer.Option(False, "--json"), -): - with open(lens, "r", encoding="utf-8") as f: - lens_data = json.load(f) - with open(repo, "r", encoding="utf-8") as f: - repo_map = json.load(f) - info = explain_node(qname, lens_data, repo_map) - if json_out: - print(json.dumps(info, indent=2)) - return - print(info.get("qname")) - print(" reason:") - for r in info.get("reason", []): - k, v = next(iter(r.items())) - print(f" - {k}: {v}") - edges = info.get("edges", {}) - print(" edges:") - print(" callers:", ", ".join(edges.get("callers", []))) - print(" callees:", ", ".join(edges.get("callees", []))) - - -@app.command() -def ci( - scan_path: str = typer.Option(".", "--scan", help="Path to scan for analysis"), - output_dir: str = typer.Option( - "ci-artifacts", "--output", "-o", help="Output directory for CI artifacts" - ), - fail_on_issues: bool = typer.Option( - True, "--fail-on-issues", help="Fail CI on understanding issues" - ), - generate_report: bool = typer.Option(True, "--report", help="Generate CI report"), -): - """Run Understand-First analysis for CI/CD pipeline with enhanced reporting.""" - console = Console() - - try: - # Create output directory - os.makedirs(output_dir, exist_ok=True) - - # Step 1: Scan repository - console.print("[bold blue]🔍 Scanning repository...[/bold blue]") - repo_map_path = os.path.join(output_dir, "repo.json") - - with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - BarColumn(), - TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), - TimeElapsedColumn(), - console=console, - ) as progress: - scan_task = progress.add_task("Scanning codebase...", total=100) - progress.update(scan_task, advance=50) - - # Run scan - result = build_python_map(pathlib.Path(scan_path)) - progress.update(scan_task, advance=50) - - # Write repo map - with open(repo_map_path, "w", encoding="utf-8") as f: - json.dump(result, f, indent=2) - - console.print(f"[green]✓ Repository map generated: {repo_map_path}[/green]") - - # Step 2: Generate lens - console.print("[bold blue]🎯 Generating understanding lens...[/bold blue]") - lens_path = os.path.join(output_dir, "lens.json") - - # Use first few functions as seeds - functions = list(result.get("functions", {}).keys()) - seeds = functions[:3] if functions else [] - - if seeds: - lens = lens_from_seeds(seeds, result) - rank_by_error_proximity(lens) - - with open(lens_path, "w", encoding="utf-8") as f: - json.dump(lens, f, indent=2) - - console.print(f"[green]✓ Understanding lens generated: {lens_path}[/green]") - else: - console.print("[yellow]⚠ No functions found for lens generation[/yellow]") - lens = {"functions": {}, "lens": {"seeds": []}} - - # Step 3: Generate tour - console.print("[bold blue]🗺️ Generating understanding tour...[/bold blue]") - tour_path = os.path.join(output_dir, "tour.md") - - if lens.get("functions"): - tour_md = write_tour_md(lens) - with open(tour_path, "w", encoding="utf-8") as f: - f.write(tour_md) - console.print(f"[green]✓ Understanding tour generated: {tour_path}[/green]") - else: - console.print("[yellow]⚠ No lens data for tour generation[/yellow]") - - # Step 4: Generate CI report - if generate_report: - console.print("[bold blue]📊 Generating CI report...[/bold blue]") - report_path = os.path.join(output_dir, "ci-report.md") - - report_content = generate_ci_report(result, lens, scan_path) - with open(report_path, "w", encoding="utf-8") as f: - f.write(report_content) - console.print(f"[green]✓ CI report generated: {report_path}[/green]") - - # Step 5: Check for issues - issues = [] - functions_count = len(result.get("functions", {})) - - if functions_count == 0: - issues.append("No functions found in codebase") - - # Check for high complexity functions - high_complexity = [] - for func_name, func_data in result.get("functions", {}).items(): - complexity = func_data.get("complexity", 0) - if complexity > 10: - high_complexity.append(f"{func_name} (complexity: {complexity})") - - if high_complexity: - issues.append(f"High complexity functions found: {', '.join(high_complexity[:3])}") - - # Check for side effects - side_effects = [] - for func_name, func_data in result.get("functions", {}).items(): - effects = func_data.get("side_effects", []) - if effects: - side_effects.append(f"{func_name}: {', '.join(effects)}") - - if side_effects: - issues.append(f"Functions with side effects: {', '.join(side_effects[:3])}") - - # Step 6: Report results - if issues: - console.print("\n[yellow]⚠ Issues detected:[/yellow]") - for issue in issues: - console.print(f" • {issue}") - - if fail_on_issues: - console.print("\n[red]❌ CI failed due to understanding issues[/red]") - raise typer.Exit(1) - else: - console.print("\n[yellow]⚠ CI completed with warnings[/yellow]") - else: - console.print("\n[green]✅ CI completed successfully - no issues detected[/green]") - - # Step 7: Generate summary - summary = { - "scan_path": scan_path, - "output_dir": output_dir, - "functions_analyzed": functions_count, - "issues_found": len(issues), - "artifacts_generated": [ - "repo.json", - "lens.json" if seeds else None, - "tour.md" if lens.get("functions") else None, - "ci-report.md" if generate_report else None, - ], - "issues": issues, - } - - # Remove None values - summary["artifacts_generated"] = [ - a for a in summary["artifacts_generated"] if a is not None - ] - - summary_path = os.path.join(output_dir, "ci-summary.json") - with open(summary_path, "w", encoding="utf-8") as f: - json.dump(summary, f, indent=2) - - console.print(f"\n[green]✓ CI analysis completed. Summary: {summary_path}[/green]") - - # Track TTU metric - ttu_record( - "ci_completed", - { - "functions_analyzed": functions_count, - "issues_found": len(issues), - "artifacts_generated": len(summary["artifacts_generated"]), - }, - ) - - except Exception as e: - console.print(f"[red]Error during CI analysis:[/red] {str(e)}") - raise typer.Exit(1) - - -def generate_ci_report(repo_map, lens, scan_path): - """Generate comprehensive CI report.""" - functions = repo_map.get("functions", {}) - functions_count = len(functions) - - # Calculate metrics - total_complexity = sum(func.get("complexity", 0) for func in functions.values()) - avg_complexity = total_complexity / functions_count if functions_count > 0 else 0 - - high_complexity_count = sum(1 for func in functions.values() if func.get("complexity", 0) > 10) - side_effects_count = sum(1 for func in functions.values() if func.get("side_effects", [])) - - # Generate report - report = f"""# Understand-First CI Report - -## Analysis Summary - -- **Scan Path**: `{scan_path}` -- **Functions Analyzed**: {functions_count} -- **Total Complexity**: {total_complexity} -- **Average Complexity**: {avg_complexity:.1f} -- **High Complexity Functions**: {high_complexity_count} -- **Functions with Side Effects**: {side_effects_count} - -## Understanding Lens - -- **Seeds**: {len(lens.get('lens', {}).get('seeds', []))} -- **Functions in Lens**: {len(lens.get('functions', {}))} - -## Recommendations - -""" - - if high_complexity_count > 0: - report += f"- Consider refactoring {high_complexity_count} high-complexity functions\n" - - if side_effects_count > 0: - report += f"- Review {side_effects_count} functions with side effects\n" - - if avg_complexity > 5: - report += "- Overall complexity is high - consider architectural improvements\n" - - if functions_count == 0: - report += "- No functions found - check scan path and file patterns\n" - - report += """ -## Next Steps - -1. Review the understanding tour for key insights -2. Address any high-complexity functions -3. Document side effects and their implications -4. Update understanding artifacts as code evolves - -*Generated by Understand-First CI* -""" - - return report - - -@app.command() -def wizard( - scan_path: str = typer.Option(".", "--scan", help="Path to scan for analysis"), - interactive: bool = typer.Option(True, "--interactive", help="Run in interactive mode"), -): - """Interactive wizard to guide users through understanding analysis.""" - console = Console() - - if not interactive: - console.print("[yellow]Non-interactive mode - running basic analysis[/yellow]") - # Run basic analysis - result = build_python_map(pathlib.Path(scan_path)) - console.print( - f"[green]Analysis complete. Found {len(result.get('functions', {}))} functions.[/green]" - ) - return - - # Enhanced welcome with better formatting - console.print( - Panel.fit( - "[bold blue]🧙 Welcome to the Understand-First Wizard![/bold blue]\n\n" - "This wizard will guide you through understanding your codebase step by step.\n" - "We'll help you identify key functions, understand their relationships,\n" - "and create a personalized learning path.", - title="Getting Started", - border_style="blue", - ) - ) - - try: - # Step 1: Enhanced repository scanning - console.print("\n[bold cyan]Step 1: Scanning your codebase...[/bold cyan]") - - with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - BarColumn(), - TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), - TimeElapsedColumn(), - console=console, - ) as progress: - scan_task = progress.add_task("Analyzing code structure...", total=100) - - # Simulate more realistic progress - progress.update(scan_task, advance=20) - time.sleep(0.5) - - progress.update(scan_task, advance=30) - result = build_python_map(pathlib.Path(scan_path)) - - progress.update(scan_task, advance=30) - time.sleep(0.3) - - progress.update(scan_task, advance=20) - - functions = result.get("functions", {}) - - # Enhanced success message with stats - if functions: - high_complexity = sum(1 for f in functions.values() if f.get("complexity", 0) > 5) - with_side_effects = sum(1 for f in functions.values() if f.get("side_effects", [])) - - console.print( - Panel.fit( - f"[green]✓ Analysis complete![/green]\n\n" - f"• [bold]{len(functions)}[/bold] functions found\n" - f"• [bold]{high_complexity}[/bold] high complexity functions\n" - f"• [bold]{with_side_effects}[/bold] functions with side effects", - title="Analysis Results", - border_style="green", - ) - ) - else: - console.print( - Panel.fit( - "[yellow]⚠ No functions found in the specified path[/yellow]\n\n" - "This could mean:\n" - "• The path doesn't contain Python files\n" - "• The files don't have function definitions\n" - "• There's an issue with the file permissions", - title="No Functions Found", - border_style="yellow", - ) - ) - return - - # Step 2: Enhanced function overview with better formatting - console.print("\n[bold cyan]Step 2: Function Overview[/bold cyan]") - - # Create a table for better function display - table = Table(title="Functions in your codebase") - table.add_column("Name", style="cyan", no_wrap=True) - table.add_column("Complexity", justify="center", style="magenta") - table.add_column("Side Effects", justify="center", style="yellow") - table.add_column("Lines", justify="center", style="green") - - for func_name, func_data in list(functions.items())[:10]: - complexity = func_data.get("complexity", 0) - side_effects = func_data.get("side_effects", []) - lines = func_data.get("lines", 0) - - complexity_str = str(complexity) - if complexity > 5: - complexity_str = f"[red]{complexity}[/red]" - elif complexity > 3: - complexity_str = f"[yellow]{complexity}[/yellow]" - - effects_str = str(len(side_effects)) if side_effects else "0" - if side_effects: - effects_str = f"[yellow]{len(side_effects)}[/yellow]" - - table.add_row(func_name, complexity_str, effects_str, str(lines)) - - console.print(table) - - if len(functions) > 10: - console.print(f"\n[yellow]... and {len(functions) - 10} more functions[/yellow]") - - # Step 3: Enhanced seed selection with better UX - console.print("\n[bold cyan]Step 3: Select Seed Functions[/bold cyan]") - console.print( - "Choose functions to start your understanding journey. These will be your 'seeds'.\n" - "Seeds help us understand which functions are most important to you.\n" - ) - - seeds = [] - while True: - if seeds: - console.print(f"[green]Current seeds: {', '.join(seeds)}[/green]") - else: - console.print("[yellow]No seeds selected yet[/yellow]") - - choice = typer.prompt( - "Enter function name to add as seed (or 'done' to continue, 'list' to see all functions, 'help' for tips)", - default="done", - ) - - if choice.lower() == "done": - break - elif choice.lower() == "list": - console.print("\n[bold]All functions:[/bold]") - for i, func_name in enumerate(functions.keys(), 1): - complexity = functions[func_name].get("complexity", 0) - console.print(f" {i:2d}. {func_name} (complexity: {complexity})") - console.print() - continue - elif choice.lower() == "help": - console.print("\n[bold]Tips for selecting seeds:[/bold]") - console.print("• Choose functions you're most interested in understanding") - console.print("• Pick functions that seem central to your codebase") - console.print("• Consider functions with high complexity or side effects") - console.print("• You can always add more seeds later\n") - continue - elif choice in functions: - if choice not in seeds: - seeds.append(choice) - console.print(f"[green]✓ Added '{choice}' as seed[/green]") - else: - console.print(f"[yellow]'{choice}' is already a seed[/yellow]") - else: - console.print( - f"[red]Function '{choice}' not found. Try 'list' to see available functions.[/red]" - ) - - if not seeds: - # Auto-select first few functions with better explanation - seeds = list(functions.keys())[:3] - console.print(f"[yellow]No seeds selected. Using first 3 functions: {seeds}[/yellow]") - console.print( - "[dim]You can always run the wizard again to select different seeds[/dim]" - ) - - console.print(f"\n[green]✓ Selected seeds: {seeds}[/green]\n") - - # Step 4: Enhanced lens generation with better progress - console.print("[bold cyan]Step 4: Generating Understanding Lens...[/bold cyan]") - - with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - BarColumn(), - TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), - TimeElapsedColumn(), - console=console, - ) as progress: - lens_task = progress.add_task("Building understanding lens...", total=100) - - progress.update(lens_task, advance=25) - time.sleep(0.3) - - progress.update(lens_task, advance=25) - lens = lens_from_seeds(seeds, result) - - progress.update(lens_task, advance=25) - rank_by_error_proximity(lens) - - progress.update(lens_task, advance=25) - - lens_functions = lens.get("functions", {}) - - # Enhanced lens results display - console.print( - Panel.fit( - f"[green]✓ Generated understanding lens![/green]\n\n" - f"• [bold]{len(lens_functions)}[/bold] functions in your lens\n" - f"• Functions ranked by importance and error proximity\n" - f"• Ready for tour generation", - title="Lens Generated", - border_style="green", - ) - ) - - # Step 5: Enhanced lens results with better formatting - console.print("\n[bold cyan]Step 5: Understanding Lens Results[/bold cyan]") - console.print("Functions in your understanding lens (ranked by importance):\n") - - # Create a table for lens results - lens_table = Table(title="Understanding Lens Functions") - lens_table.add_column("Rank", justify="center", style="cyan") - lens_table.add_column("Function", style="cyan", no_wrap=True) - lens_table.add_column("Complexity", justify="center", style="magenta") - lens_table.add_column("Side Effects", justify="center", style="yellow") - lens_table.add_column("Importance", justify="center", style="green") - - for i, (func_name, func_data) in enumerate(lens_functions.items(), 1): - complexity = func_data.get("complexity", 0) - side_effects = func_data.get("side_effects", []) - importance = func_data.get("importance", 0) - - complexity_str = str(complexity) - if complexity > 5: - complexity_str = f"[red]{complexity}[/red]" - elif complexity > 3: - complexity_str = f"[yellow]{complexity}[/yellow]" - - effects_str = str(len(side_effects)) if side_effects else "0" - if side_effects: - effects_str = f"[yellow]{len(side_effects)}[/yellow]" - - importance_str = f"{importance:.1f}" if importance else "N/A" - - lens_table.add_row(str(i), func_name, complexity_str, effects_str, importance_str) - - console.print(lens_table) - - # Step 6: Enhanced tour generation - console.print("\n[bold cyan]Step 6: Generating Understanding Tour...[/bold cyan]") - - if lens_functions: - with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - BarColumn(), - TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), - TimeElapsedColumn(), - console=console, - ) as progress: - tour_task = progress.add_task("Generating understanding tour...", total=100) - - progress.update(tour_task, advance=50) - tour_md = write_tour_md(lens) - - progress.update(tour_task, advance=30) - tour_path = "understanding_tour.md" - with open(tour_path, "w", encoding="utf-8") as f: - f.write(tour_md) - - progress.update(tour_task, advance=20) - - console.print( - Panel.fit( - f"[green]✓ Tour generated successfully![/green]\n\n" - f"• File: [bold]{tour_path}[/bold]\n" - f"• Contains step-by-step guidance\n" - f"• Ready for review and learning", - title="Tour Generated", - border_style="green", - ) - ) - else: - console.print( - Panel.fit( - "[yellow]⚠ No functions in lens for tour generation[/yellow]\n\n" - "This usually means the selected seeds didn't have enough\n" - "connections to other functions in your codebase.", - title="No Tour Generated", - border_style="yellow", - ) - ) - - # Step 7: Enhanced summary and next steps - console.print("\n[bold green]🎉 Wizard Complete![/bold green]") - - # Create a summary table - summary_table = Table(title="What you've accomplished") - summary_table.add_column("Metric", style="cyan") - summary_table.add_column("Value", style="green") - - summary_table.add_row("Functions analyzed", str(len(functions))) - summary_table.add_row("Seeds selected", str(len(seeds))) - summary_table.add_row("Lens functions", str(len(lens_functions))) - summary_table.add_row("Tour generated", "Yes" if lens_functions else "No") - - console.print(summary_table) - - console.print("\n[bold cyan]Next steps:[/bold cyan]") - console.print(" • Review the understanding tour to understand your code") - console.print(" • Use 'u map' to visualize the code structure") - console.print(" • Use 'u diff' to track changes over time") - console.print(" • Use 'u ci' to integrate with your CI/CD pipeline") - console.print(" • Use 'u tui' for an interactive exploration experience") - - # Track TTU metric - ttu_record( - "wizard_completed", - { - "functions_analyzed": len(functions), - "seeds_selected": len(seeds), - "lens_functions": len(lens_functions), - "tour_generated": bool(lens_functions), - }, - ) - - except KeyboardInterrupt: - console.print("\n[yellow]Wizard interrupted by user[/yellow]") - console.print("[dim]You can run 'u wizard' again anytime to continue[/dim]") - except Exception as e: - console.print( - Panel.fit( - f"[red]Error during wizard:[/red] {str(e)}\n\n" - "This might be due to:\n" - "• File permission issues\n" - "• Corrupted code files\n" - "• Missing dependencies\n\n" - "Try running with --interactive=false for basic analysis", - title="Wizard Error", - border_style="red", - ) - ) - raise typer.Exit(1) - - -@app.command() -def tui( - scan_path: str = typer.Option(".", "--scan", help="Path to scan for analysis"), -): - """Launch interactive Text User Interface for code understanding.""" - console = Console() - - try: - # Enhanced welcome with better formatting - console.print( - Panel.fit( - "[bold blue]🚀 Understand-First TUI[/bold blue]\n\n" - "Interactive exploration of your codebase.\n" - "Navigate through functions, understand relationships,\n" - "and generate personalized learning paths.", - title="Welcome", - border_style="blue", - ) - ) - console.print("[dim]Press Ctrl+C to exit at any time[/dim]\n") - - # Enhanced data loading with progress - with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - BarColumn(), - TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), - TimeElapsedColumn(), - console=console, - ) as progress: - load_task = progress.add_task("Loading codebase...", total=100) - - progress.update(load_task, advance=30) - time.sleep(0.2) - - progress.update(load_task, advance=40) - result = build_python_map(pathlib.Path(scan_path)) - - progress.update(load_task, advance=30) - - functions = result.get("functions", {}) - - if functions: - high_complexity = sum(1 for f in functions.values() if f.get("complexity", 0) > 5) - with_side_effects = sum(1 for f in functions.values() if f.get("side_effects", [])) - - console.print( - Panel.fit( - f"[green]✓ Codebase loaded successfully![/green]\n\n" - f"• [bold]{len(functions)}[/bold] functions available\n" - f"• [bold]{high_complexity}[/bold] high complexity functions\n" - f"• [bold]{with_side_effects}[/bold] functions with side effects", - title="Ready to Explore", - border_style="green", - ) - ) - else: - console.print( - Panel.fit( - "[yellow]⚠ No functions found in the specified path[/yellow]\n\n" - "This could mean:\n" - "• The path doesn't contain Python files\n" - "• The files don't have function definitions\n" - "• There's an issue with the file permissions", - title="No Functions Found", - border_style="yellow", - ) - ) - return - - # Enhanced main TUI loop - while True: - console.print("\n[bold cyan]Main Menu[/bold cyan]") - - # Create a menu table for better visual appeal - menu_table = Table(show_header=False, box=None) - menu_table.add_column("Option", style="cyan", width=3) - menu_table.add_column("Description", style="white") - - menu_table.add_row("1", "📋 View function list") - menu_table.add_row("2", "🔍 Search functions") - menu_table.add_row("3", "🔬 Generate understanding lens") - menu_table.add_row("4", "📊 View function details") - menu_table.add_row("5", "📚 Generate tour") - menu_table.add_row("6", "💾 Export data") - menu_table.add_row("7", "📈 View metrics") - menu_table.add_row("0", "🚪 Exit") - - console.print(menu_table) - - try: - choice = typer.prompt("\nSelect option", type=int, default=0) - except (ValueError, typer.Abort): - console.print("[red]Invalid input. Please enter a number.[/red]") - continue - - if choice == 0: - console.print( - Panel.fit( - "[yellow]Thanks for using Understand-First TUI![/yellow]\n\n" - "Keep exploring your codebase and building understanding!", - title="Goodbye", - border_style="yellow", - ) - ) - break - elif choice == 1: - show_function_list(console, functions) - elif choice == 2: - search_functions(console, functions) - elif choice == 3: - generate_lens_interactive(console, functions, result) - elif choice == 4: - view_function_details(console, functions) - elif choice == 5: - generate_tour_interactive(console, functions, result) - elif choice == 6: - export_data_interactive(console, result) - elif choice == 7: - show_metrics(console, result) - else: - console.print( - Panel.fit( - "[red]Invalid option. Please try again.[/red]\n\n" - "Select a number between 0-7", - title="Invalid Selection", - border_style="red", - ) - ) - - # Track TTU metric - ttu_record("tui_session", {"functions_loaded": len(functions)}) - - except KeyboardInterrupt: - console.print("\n[yellow]TUI interrupted by user[/yellow]") - console.print("[dim]You can run 'u tui' again anytime to continue[/dim]") - except Exception as e: - console.print( - Panel.fit( - f"[red]Error in TUI:[/red] {str(e)}\n\n" - "This might be due to:\n" - "• File permission issues\n" - "• Corrupted code files\n" - "• Missing dependencies", - title="TUI Error", - border_style="red", - ) - ) - raise typer.Exit(1) - - -def show_function_list(console, functions): - """Show paginated function list with enhanced formatting.""" - console.print("\n[bold cyan]Function List[/bold cyan]") - - page_size = 20 - total_pages = (len(functions) + page_size - 1) // page_size - page = 0 - - while True: - start_idx = page * page_size - end_idx = min(start_idx + page_size, len(functions)) - - # Create a table for better function display - table = Table(title=f"Functions (Page {page + 1} of {total_pages})") - table.add_column("Index", justify="center", style="cyan", width=5) - table.add_column("Function", style="cyan", no_wrap=True) - table.add_column("Complexity", justify="center", style="magenta", width=10) - table.add_column("Side Effects", justify="center", style="yellow", width=12) - table.add_column("Lines", justify="center", style="green", width=6) - - for i, (func_name, func_data) in enumerate(list(functions.items())[start_idx:end_idx]): - complexity = func_data.get("complexity", 0) - side_effects = func_data.get("side_effects", []) - lines = func_data.get("lines", 0) - - complexity_str = str(complexity) - if complexity > 5: - complexity_str = f"[red]{complexity}[/red]" - elif complexity > 3: - complexity_str = f"[yellow]{complexity}[/yellow]" - - effects_str = str(len(side_effects)) if side_effects else "0" - if side_effects: - effects_str = f"[yellow]{len(side_effects)}[/yellow]" - - table.add_row( - str(start_idx + i + 1), func_name, complexity_str, effects_str, str(lines) - ) - - console.print(table) - - console.print("\n[dim]Commands: n=next, p=previous, q=quit[/dim]") - cmd = typer.prompt("", default="q") - - if cmd.lower() == "q": - break - elif cmd.lower() == "n" and page < total_pages - 1: - page += 1 - elif cmd.lower() == "p" and page > 0: - page -= 1 - else: - console.print("[red]Invalid command. Use n, p, or q.[/red]") - - -def show_metrics(console, result): - """Show codebase metrics and statistics.""" - functions = result.get("functions", {}) - - if not functions: - console.print("[yellow]No functions to analyze metrics for.[/yellow]") - return - - # Calculate metrics - total_functions = len(functions) - high_complexity = sum(1 for f in functions.values() if f.get("complexity", 0) > 5) - with_side_effects = sum(1 for f in functions.values() if f.get("side_effects", [])) - avg_complexity = sum(f.get("complexity", 0) for f in functions.values()) / total_functions - total_lines = sum(f.get("lines", 0) for f in functions.values()) - - # Create metrics table - metrics_table = Table(title="Codebase Metrics") - metrics_table.add_column("Metric", style="cyan") - metrics_table.add_column("Value", style="green") - metrics_table.add_column("Percentage", style="yellow") - - metrics_table.add_row("Total Functions", str(total_functions), "100%") - metrics_table.add_row( - "High Complexity", str(high_complexity), f"{high_complexity/total_functions*100:.1f}%" - ) - metrics_table.add_row( - "With Side Effects", str(with_side_effects), f"{with_side_effects/total_functions*100:.1f}%" - ) - metrics_table.add_row("Average Complexity", f"{avg_complexity:.1f}", "-") - metrics_table.add_row("Total Lines", str(total_lines), "-") - - console.print(metrics_table) - - # Show complexity distribution - complexity_dist = {} - for f in functions.values(): - complexity = f.get("complexity", 0) - if complexity <= 2: - complexity_dist["Low (1-2)"] = complexity_dist.get("Low (1-2)", 0) + 1 - elif complexity <= 5: - complexity_dist["Medium (3-5)"] = complexity_dist.get("Medium (3-5)", 0) + 1 - else: - complexity_dist["High (6+)"] = complexity_dist.get("High (6+)", 0) + 1 - - console.print("\n[bold cyan]Complexity Distribution[/bold cyan]") - for level, count in complexity_dist.items(): - percentage = count / total_functions * 100 - console.print(f" {level}: {count} functions ({percentage:.1f}%)") - - # Show top complex functions - top_complex = sorted(functions.items(), key=lambda x: x[1].get("complexity", 0), reverse=True)[ - :5 - ] - if top_complex: - console.print("\n[bold cyan]Top 5 Most Complex Functions[/bold cyan]") - for i, (name, data) in enumerate(top_complex, 1): - complexity = data.get("complexity", 0) - console.print(f" {i}. {name} (complexity: {complexity})") - - -def search_functions(console, functions): - """Search functions by name or complexity with enhanced formatting.""" - console.print("\n[bold cyan]Function Search[/bold cyan]") - console.print("Search by function name or complexity threshold\n") - - query = typer.prompt("Enter search term (function name or complexity threshold)") - - if query.isdigit(): - # Search by complexity - threshold = int(query) - matches = [ - (name, data) - for name, data in functions.items() - if data.get("complexity", 0) >= threshold - ] - - if matches: - console.print( - Panel.fit( - f"[green]Found {len(matches)} functions with complexity >= {threshold}[/green]", - title="Search Results", - border_style="green", - ) - ) - - # Create a table for matches - table = Table(title=f"Functions with complexity >= {threshold}") - table.add_column("Function", style="cyan", no_wrap=True) - table.add_column("Complexity", justify="center", style="magenta", width=10) - table.add_column("Side Effects", justify="center", style="yellow", width=12) - table.add_column("Lines", justify="center", style="green", width=6) - - for name, data in matches: - complexity = data.get("complexity", 0) - side_effects = data.get("side_effects", []) - lines = data.get("lines", 0) - - complexity_str = str(complexity) - if complexity > 5: - complexity_str = f"[red]{complexity}[/red]" - elif complexity > 3: - complexity_str = f"[yellow]{complexity}[/yellow]" - - effects_str = str(len(side_effects)) if side_effects else "0" - if side_effects: - effects_str = f"[yellow]{len(side_effects)}[/yellow]" - - table.add_row(name, complexity_str, effects_str, str(lines)) - - console.print(table) - else: - console.print( - Panel.fit( - f"[yellow]No functions found with complexity >= {threshold}[/yellow]", - title="No Matches", - border_style="yellow", - ) - ) - else: - # Search by name - matches = [ - (name, data) for name, data in functions.items() if query.lower() in name.lower() - ] - - if matches: - console.print( - Panel.fit( - f"[green]Found {len(matches)} functions matching '{query}'[/green]", - title="Search Results", - border_style="green", - ) - ) - - # Create a table for matches - table = Table(title=f"Functions matching '{query}'") - table.add_column("Function", style="cyan", no_wrap=True) - table.add_column("Complexity", justify="center", style="magenta", width=10) - table.add_column("Side Effects", justify="center", style="yellow", width=12) - table.add_column("Lines", justify="center", style="green", width=6) - - for name, data in matches: - complexity = data.get("complexity", 0) - side_effects = data.get("side_effects", []) - lines = data.get("lines", 0) - - complexity_str = str(complexity) - if complexity > 5: - complexity_str = f"[red]{complexity}[/red]" - elif complexity > 3: - complexity_str = f"[yellow]{complexity}[/yellow]" - - effects_str = str(len(side_effects)) if side_effects else "0" - if side_effects: - effects_str = f"[yellow]{len(side_effects)}[/yellow]" - - table.add_row(name, complexity_str, effects_str, str(lines)) - - console.print(table) - else: - console.print( - Panel.fit( - f"[yellow]No functions found matching '{query}'[/yellow]\n\n" - "Try:\n" - "• A different search term\n" - "• A partial function name\n" - "• A complexity threshold (number)", - title="No Matches", - border_style="yellow", - ) - ) - - -def generate_lens_interactive(console, functions, result): - """Generate lens interactively.""" - console.print("\n[bold]Generate Understanding Lens[/bold]") - - # Show function selection - console.print("Select seed functions (comma-separated numbers):") - func_list = list(functions.items()) - for i, (name, data) in enumerate(func_list[:20]): # Show first 20 - complexity = data.get("complexity", 0) - console.print(f" {i+1:2d}. {name} (complexity: {complexity})") - - if len(func_list) > 20: - console.print(f" ... and {len(func_list) - 20} more") - - selection = typer.prompt("Enter function numbers (e.g., 1,3,5)", default="1,2,3") - - try: - indices = [int(x.strip()) - 1 for x in selection.split(",")] - seeds = [func_list[i][0] for i in indices if 0 <= i < len(func_list)] - - if not seeds: - console.print("[red]No valid functions selected[/red]") - return - - console.print(f"[green]Selected seeds: {seeds}[/green]") - - # Generate lens - with console.status("[bold green]Generating lens..."): - lens = lens_from_seeds(seeds, result) - rank_by_error_proximity(lens) - - lens_functions = lens.get("functions", {}) - console.print(f"[green]✓ Generated lens with {len(lens_functions)} functions[/green]") - - # Show lens results - console.print("\nLens functions (ranked by importance):") - for i, (name, data) in enumerate(lens_functions.items(), 1): - complexity = data.get("complexity", 0) - console.print(f" {i:2d}. {name} (complexity: {complexity})") - - except (ValueError, IndexError) as e: - console.print(f"[red]Invalid selection: {e}[/red]") - - -def view_function_details(console, functions): - """View detailed function information.""" - console.print("\n[bold]Function Details[/bold]") - - func_name = typer.prompt("Enter function name") - - if func_name not in functions: - console.print(f"[red]Function '{func_name}' not found[/red]") - return - - data = functions[func_name] - console.print(f"\n[bold]{func_name}[/bold]") - console.print(f" Complexity: {data.get('complexity', 0)}") - console.print(f" Side effects: {', '.join(data.get('side_effects', []))}") - - edges = data.get("edges", {}) - if edges.get("callers"): - console.print(f" Callers: {', '.join(edges['callers'])}") - if edges.get("callees"): - console.print(f" Callees: {', '.join(edges['callees'])}") - - -def generate_tour_interactive(console, functions, result): - """Generate tour interactively.""" - console.print("\n[bold]Generate Understanding Tour[/bold]") - - # Check if we have a lens - seeds = typer.prompt("Enter seed functions (comma-separated)", default="") - - if not seeds: - console.print("[yellow]No seeds provided. Using first 3 functions.[/yellow]") - seeds = list(functions.keys())[:3] - else: - seeds = [s.strip() for s in seeds.split(",")] - # Validate seeds - valid_seeds = [s for s in seeds if s in functions] - if not valid_seeds: - console.print("[red]No valid seeds found[/red]") - return - seeds = valid_seeds - - console.print(f"[green]Using seeds: {seeds}[/green]") - - # Generate lens and tour - with console.status("[bold green]Generating tour..."): - lens = lens_from_seeds(seeds, result) - rank_by_error_proximity(lens) - - if lens.get("functions"): - tour_md = write_tour_md(lens) - tour_path = "understanding_tour.md" - with open(tour_path, "w", encoding="utf-8") as f: - f.write(tour_md) - console.print(f"[green]✓ Tour generated: {tour_path}[/green]") - else: - console.print("[yellow]No functions in lens for tour generation[/yellow]") - - -def export_data_interactive(console, result): - """Export data interactively.""" - console.print("\n[bold]Export Data[/bold]") - console.print("1. Export as JSON") - console.print("2. Export as Markdown") - console.print("3. Export as SVG") - - choice = typer.prompt("Select format", type=int, default=1) - - if choice == 1: - output_path = "analysis.json" - with open(output_path, "w", encoding="utf-8") as f: - json.dump(result, f, indent=2) - console.print(f"[green]✓ Exported to {output_path}[/green]") - elif choice == 2: - output_path = "analysis.md" - # Generate simple markdown report - functions = result.get("functions", {}) - with open(output_path, "w", encoding="utf-8") as f: - f.write(f"# Code Analysis Report\n\n") - f.write(f"Functions analyzed: {len(functions)}\n\n") - f.write("## Functions\n\n") - for name, data in functions.items(): - complexity = data.get("complexity", 0) - side_effects = data.get("side_effects", []) - f.write(f"- **{name}** (complexity: {complexity})\n") - if side_effects: - f.write(f" - Side effects: {', '.join(side_effects)}\n") - console.print(f"[green]✓ Exported to {output_path}[/green]") - elif choice == 3: - console.print("[yellow]SVG export not implemented in TUI mode[/yellow]") - else: - console.print("[red]Invalid choice[/red]") - - -@app.command() -def metrics( - days: int = typer.Option(30, "--days", "-d", help="Number of days to analyze"), - format: str = typer.Option("report", "--format", "-f", help="Output format: report, json, csv"), - output: str = typer.Option(None, "--output", "-o", help="Output file path"), -): - """Generate metrics report for TTU and TTFSC tracking.""" - console = Console() - - try: - from ucli.metrics.analytics import ( - get_dashboard_data, - generate_metrics_report, - export_metrics_csv, - ) - - console.print( - f"[bold blue]📊 Generating metrics report for last {days} days...[/bold blue]" - ) - - if format == "report": - report = generate_metrics_report(days) - - if output: - with open(output, "w", encoding="utf-8") as f: - f.write(report) - console.print(f"[green]✓ Metrics report saved to {output}[/green]") - else: - console.print("\n" + report) - - elif format == "json": - data = get_dashboard_data(days) - - if output: - with open(output, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2) - console.print(f"[green]✓ Metrics data saved to {output}[/green]") - else: - console.print(json.dumps(data, indent=2)) - - elif format == "csv": - output_file = output or f"metrics_export_{days}d.csv" - csv_file = export_metrics_csv(days, output_file) - console.print(f"[green]✓ Metrics CSV exported to {csv_file}[/green]") - - else: - console.print("[red]Invalid format. Use: report, json, or csv[/red]") - raise typer.Exit(1) - - except ImportError: - console.print("[red]Analytics module not available[/red]") - raise typer.Exit(1) - except Exception as e: - console.print(f"[red]Error generating metrics:[/red] {str(e)}") - raise typer.Exit(1) diff --git a/cli/ucli/metrics/ttu.py b/cli/ucli/metrics/ttu.py index 540df05..7ee9351 100644 --- a/cli/ucli/metrics/ttu.py +++ b/cli/ucli/metrics/ttu.py @@ -26,7 +26,7 @@ def weekly_report(outfile="docs/ttu.md"): for line in f: try: rows.append(json.loads(line)) - except: + except (json.JSONDecodeError, TypeError, ValueError): pass # naive: compute deltas between first map_open -> tour_run -> fixture_pass per day days = {} diff --git a/cli/ucli/pack/publish.py b/cli/ucli/pack/publish.py index 4351c54..303625a 100644 --- a/cli/ucli/pack/publish.py +++ b/cli/ucli/pack/publish.py @@ -1,18 +1,38 @@ -import os, zipfile, json, pathlib, time +import json +import os +import time +import zipfile -def make_pack(dist_dir='dist'): + +def make_pack(dist_dir="dist"): os.makedirs(dist_dir, exist_ok=True) - pack_path = os.path.join(dist_dir, 'understanding-pack.zip') - with zipfile.ZipFile(pack_path, 'w', zipfile.ZIP_DEFLATED) as z: + pack_path = os.path.join(dist_dir, "understanding-pack.zip") + with zipfile.ZipFile(pack_path, "w", zipfile.ZIP_DEFLATED) as z: + def addp(p): - if os.path.exists(p): z.write(p, p) - addp('tours/PR.md'); addp('tours/local.md') - addp('maps/delta.svg'); addp('docs/understanding-dashboard.md'); addp('docs/glossary.md') - for root, _, files in os.walk('maps'): + if os.path.exists(p): + z.write(p, p) + + addp("tours/PR.md") + addp("tours/local.md") + addp("maps/delta.svg") + addp("docs/understanding-dashboard.md") + addp("docs/glossary.md") + for root, _, files in os.walk("maps"): for f in files: - if f.endswith('.json'): z.write(os.path.join(root, f), os.path.join(root, f)) - meta = {'created': int(time.time()), 'paths': ['tours/PR.md','tours/local.md','maps/delta.svg','docs/understanding-dashboard.md','docs/glossary.md']} - with open(os.path.join(dist_dir, 'pack.json'), 'w', encoding='utf-8') as f: + if f.endswith(".json"): + z.write(os.path.join(root, f), os.path.join(root, f)) + meta = { + "created": int(time.time()), + "paths": [ + "tours/PR.md", + "tours/local.md", + "maps/delta.svg", + "docs/understanding-dashboard.md", + "docs/glossary.md", + ], + } + with open(os.path.join(dist_dir, "pack.json"), "w", encoding="utf-8") as f: json.dump(meta, f, indent=2) print(f"Pack created at {pack_path}") return pack_path diff --git a/cli/ucli/packs/pack.py b/cli/ucli/packs/pack.py index 5cbd53b..95ccd54 100644 --- a/cli/ucli/packs/pack.py +++ b/cli/ucli/packs/pack.py @@ -1,6 +1,8 @@ import zipfile + + def create_pack(lens: str, tour: str, contracts: str, output_zip: str): - with zipfile.ZipFile(output_zip, 'w', zipfile.ZIP_DEFLATED) as z: - z.write(lens, arcname='lens.json') - z.write(tour, arcname='tour.md') - z.write(contracts, arcname='contracts.yaml') + with zipfile.ZipFile(output_zip, "w", zipfile.ZIP_DEFLATED) as z: + z.write(lens, arcname="lens.json") + z.write(tour, arcname="tour.md") + z.write(contracts, arcname="contracts.yaml") diff --git a/cli/ucli/report/report.py b/cli/ucli/report/report.py index 52ff670..d8ac36b 100644 --- a/cli/ucli/report/report.py +++ b/cli/ucli/report/report.py @@ -1,18 +1,59 @@ -from typing import Dict, Any from collections import Counter +from typing import Any -def make_report_md(system_map: Dict[str, Any]) -> str: - fns = system_map.get('functions', {}) +from ucli.analyzers.map_meta import complexity_semantics_note, map_fidelity_label + + +def make_report_md(system_map: dict[str, Any]) -> str: + fns = system_map.get("functions", {}) calls = Counter() - for q, meta in fns.items(): - for c in meta.get('calls', []): + for _q, meta in fns.items(): + for c in meta.get("calls", []): calls[c] += 1 hotspots = calls.most_common(10) - md = ["# Understanding Report", "", "## Hotspots (most called symbols)"] + md = ["# Understanding Report", ""] + + analyzer = system_map.get("analyzer") + kind = system_map.get("complexity_kind") + fidelity = map_fidelity_label(system_map) + md.append("## Map fidelity") + md.append(f"- Language: `{system_map.get('language', 'unknown')}`") + if analyzer: + md.append(f"- Analyzer: `{analyzer}`") + md.append(f"- Fidelity: `{fidelity}`") + if kind: + md.append(f"- Complexity kind: `{kind}`") + md.append(f"- {complexity_semantics_note(system_map)}") + md.append("") + + md.append("## Hotspots (most called symbols)") for name, count in hotspots: md.append(f"- `{name}` — {count} inbound calls") - md.append("\n## Next steps\n- Confirm invariants for hotspots.\n- Create/run a minimal fixture on the hot path.") + + # High-confidence concrete class returns (factory → obj.method edges when unique). + typed_returns = sorted( + (qn, meta["return_type"]) + for qn, meta in fns.items() + if isinstance(meta.get("return_type"), str) and meta["return_type"] + ) + if typed_returns: + md.append("") + md.append("## Inferred return types (high-confidence)") + md.append( + "Concrete class names used for `obj = factory(); obj.method` edges when " + "uniquely gated (including imported / `mod.Class` origins). " + "Ambiguous or multi-return factories omit `return_type`." + ) + for qn, rt in typed_returns[:25]: + md.append(f"- `{qn}` → `{rt}`") + if len(typed_returns) > 25: + md.append(f"- … and {len(typed_returns) - 25} more") + + md.append( + "\n## Next steps\n- Confirm invariants for hotspots.\n- Create/run a minimal fixture on the hot path." + ) return "\n".join(md) -def suggest_fixture(system_map: Dict[str, Any]) -> str: + +def suggest_fixture(system_map: dict[str, Any]) -> str: return "# Fill a minimal hot-path fixture here (see lens and trace outputs).\n" diff --git a/cli/ucli/trace/pytrace.py b/cli/ucli/trace/pytrace.py index 758a77b..04df673 100644 --- a/cli/ucli/trace/pytrace.py +++ b/cli/ucli/trace/pytrace.py @@ -1,25 +1,100 @@ -import sys, time, importlib.util -from typing import Any, Dict, List +"""Runtime call tracing and static error analysis. + +Security note +------------- +``run_callable_with_trace`` intentionally executes user Python in an **isolated +subprocess** (never ``exec_module`` in the CLI process). Residual risk: the +child still runs arbitrary target code — that is required for runtime tracing. +Prefer ``analyze_errors_static`` (AST-only) when execution is not needed. + +Hardening: + * Target path must resolve under an allowlisted root (cwd by default, plus + ``UF_TRACE_ALLOW_ROOTS`` pathsep-separated dirs). Symlinks that escape the + allowlist are refused after ``resolve()``. + * ``func_name`` must be a simple Python identifier; a denylist blocks + high-risk builtins (``eval``, ``exec``, ``__import__``, …). + * Worker applies soft CPU/address limits on POSIX when ``resource`` is + available. On Windows, best-effort Job Object memory limit via ctypes + (see residual risks in ``docs/SECURITY.md``). +""" + +from __future__ import annotations + import ast +import json +import os +import re +import subprocess # nosec B404 # intentional: isolated trace worker, fixed argv, no shell +import sys +import tempfile +from pathlib import Path +from typing import Any + +# Worker script executed in a subprocess. Kept as a string so the parent module +# never imports/executes the target file in-process. +_WORKER = r""" +import importlib.util +import json +import sys +import time + +def _apply_limits(): + try: + import resource + except ImportError: + return + # Soft caps only — demo/trace workloads should stay small. + try: + resource.setrlimit(resource.RLIMIT_CPU, (30, 30)) + except (ValueError, OSError): + pass + try: + # ~512 MiB address space + resource.setrlimit(resource.RLIMIT_AS, (512 * 1024 * 1024, 512 * 1024 * 1024)) + except (ValueError, OSError, AttributeError): + pass + +def _coerce(x): + if x is None: + return None + try: + return int(x) + except (TypeError, ValueError): + return x +def main(): + _apply_limits() + pyfile, func_name = sys.argv[1], sys.argv[2] + a = sys.argv[3] if len(sys.argv) > 3 else None + b = sys.argv[4] if len(sys.argv) > 4 else None + if a == "": + a = None + if b == "": + b = None -def run_callable_with_trace(pyfile: str, func_name: str, a=None, b=None) -> Dict[str, Any]: - spec = importlib.util.spec_from_file_location("_mod", pyfile) + spec = importlib.util.spec_from_file_location("_uf_trace_mod", pyfile) if spec is None or spec.loader is None: - return {"error": "could not load module", "events": []} + print(json.dumps({"error": "could not load module", "events": []})) + return 1 mod = importlib.util.module_from_spec(spec) + # INTENTIONAL: execute user module only inside this subprocess worker. spec.loader.exec_module(mod) + target = getattr(mod, func_name, None) + if target is None or not callable(target): + print(json.dumps({"error": f"callable not found: {func_name}", "events": []})) + return 1 - target = getattr(mod, func_name) - events: List[Dict[str, Any]] = [] + events = [] start = time.time() def tracer(frame, event, arg): if event == "call": code = frame.f_code - fn = code.co_name - file = code.co_filename - events.append({"type": "call", "func": fn, "file": file}) + events.append({ + "type": "call", + "func": code.co_name, + "file": code.co_filename, + }) return tracer sys.setprofile(tracer) @@ -30,17 +105,252 @@ def tracer(frame, event, arg): target(_coerce(a)) else: target(_coerce(a), _coerce(b)) + except Exception as exc: + sys.setprofile(None) + print(json.dumps({ + "error": f"{type(exc).__name__}: {exc}", + "events": events, + "duration_sec": time.time() - start, + })) + return 1 finally: sys.setprofile(None) - end = time.time() - return {"events": events, "duration_sec": end - start} + print(json.dumps({"events": events, "duration_sec": time.time() - start})) + return 0 + +if __name__ == "__main__": + raise SystemExit(main()) +""" + +# Top-level getattr targets that must never be invoked via `u trace module`. +_FUNC_DENYLIST = frozenset( + { + "eval", + "exec", + "compile", + "__import__", + "breakpoint", + "input", + "exit", + "quit", + "help", + "memoryview", + "open", + } +) +_FUNC_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + + +def _allow_roots() -> list[Path]: + roots = [Path.cwd()] + extra = os.environ.get("UF_TRACE_ALLOW_ROOTS", "") + if extra: + for part in extra.split(os.pathsep): + part = part.strip() + if part: + roots.append(Path(part)) + return roots + + +def _path_is_allowed(target: Path, roots: list[Path] | None = None) -> bool: + """True when ``target`` resolves under at least one allowlisted root.""" + try: + resolved = target.resolve() + except OSError: + return False + # Refuse non-.py targets after resolve (covers symlink renames). + if resolved.suffix.lower() != ".py": + return False + for root in roots or _allow_roots(): + try: + root_res = root.resolve() + except OSError: + continue + try: + resolved.relative_to(root_res) + return True + except ValueError: + continue + return False + + +def _validate_func_name(func_name: str) -> str | None: + """Return an error message if ``func_name`` is unsafe; else None.""" + if not func_name or not _FUNC_NAME_RE.match(func_name): + return ( + "trace refused: func must be a simple Python identifier (top-level callable name only)" + ) + if func_name in _FUNC_DENYLIST or func_name.startswith("__"): + return f"trace refused: callable '{func_name}' is denylisted" + return None + + +def _windows_assign_job_memory_limit( + proc: subprocess.Popen, limit_bytes: int = 512 * 1024 * 1024 +) -> None: + """Best-effort Windows Job Object memory cap for the trace child process. -def analyze_errors_static(pyfile: str) -> Dict[str, Any]: - src = open(pyfile, "r", encoding="utf-8", errors="ignore").read() + Failures are silent — tracing still proceeds with timeout-only containment. + """ + if sys.platform != "win32": + return + try: + import ctypes + from ctypes import wintypes + except ImportError: + return + + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + + JobObjectExtendedLimitInformation = 9 + JOB_OBJECT_LIMIT_PROCESS_MEMORY = 0x00000100 + + class IO_COUNTERS(ctypes.Structure): + _fields_ = [ + ("ReadOperationCount", ctypes.c_uint64), + ("WriteOperationCount", ctypes.c_uint64), + ("OtherOperationCount", ctypes.c_uint64), + ("ReadTransferCount", ctypes.c_uint64), + ("WriteTransferCount", ctypes.c_uint64), + ("OtherTransferCount", ctypes.c_uint64), + ] + + class JOBOBJECT_BASIC_LIMIT_INFORMATION(ctypes.Structure): + _fields_ = [ + ("PerProcessUserTimeLimit", ctypes.c_int64), + ("PerJobUserTimeLimit", ctypes.c_int64), + ("LimitFlags", wintypes.DWORD), + ("MinimumWorkingSetSize", ctypes.c_size_t), + ("MaximumWorkingSetSize", ctypes.c_size_t), + ("ActiveProcessLimit", wintypes.DWORD), + ("Affinity", ctypes.c_size_t), + ("PriorityClass", wintypes.DWORD), + ("SchedulingClass", wintypes.DWORD), + ] + + class JOBOBJECT_EXTENDED_LIMIT_INFORMATION(ctypes.Structure): + _fields_ = [ + ("BasicLimitInformation", JOBOBJECT_BASIC_LIMIT_INFORMATION), + ("IoInfo", IO_COUNTERS), + ("ProcessMemoryLimit", ctypes.c_size_t), + ("JobMemoryLimit", ctypes.c_size_t), + ("PeakProcessMemoryUsed", ctypes.c_size_t), + ("PeakJobMemoryUsed", ctypes.c_size_t), + ] + + handle = None + try: + handle = kernel32.CreateJobObjectW(None, None) + if not handle: + return + info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION() + info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_PROCESS_MEMORY + info.ProcessMemoryLimit = limit_bytes + ok = kernel32.SetInformationJobObject( + handle, + JobObjectExtendedLimitInformation, + ctypes.byref(info), + ctypes.sizeof(info), + ) + if not ok: + return + # PROCESS_SET_QUOTA | PROCESS_TERMINATE + PROCESS_SET_QUOTA = 0x0100 + PROCESS_TERMINATE = 0x0001 + proc_handle = kernel32.OpenProcess(PROCESS_SET_QUOTA | PROCESS_TERMINATE, False, proc.pid) + if not proc_handle: + return + try: + kernel32.AssignProcessToJobObject(handle, proc_handle) + finally: + kernel32.CloseHandle(proc_handle) + # Keep job handle alive until process exits by attaching to Popen. + proc._uf_job_handle = handle # type: ignore[attr-defined] + handle = None + except (OSError, AttributeError, ValueError, TypeError): + return + finally: + if handle: + try: + kernel32.CloseHandle(handle) + except OSError: + pass + + +def run_callable_with_trace(pyfile: str, func_name: str, a=None, b=None) -> dict[str, Any]: + """Run ``func_name`` from ``pyfile`` under a profile tracer in a subprocess.""" + deny = _validate_func_name(func_name) + if deny: + return {"error": deny, "events": []} + + target = Path(pyfile) + if not target.is_file(): + return {"error": f"file not found: {pyfile}", "events": []} + if "\x00" in pyfile: + return {"error": "trace refused: path contains NUL", "events": []} + if not _path_is_allowed(target): + return { + "error": ( + f"trace refused: {pyfile} is outside allowlisted roots " + "(cwd and UF_TRACE_ALLOW_ROOTS) or is not a .py file" + ), + "events": [], + } + + with tempfile.TemporaryDirectory(prefix="uf-trace-") as tmp: + worker = Path(tmp) / "_uf_trace_worker.py" + worker.write_text(_WORKER, encoding="utf-8") + cmd = [ + sys.executable, + str(worker), + str(target.resolve()), + func_name, + "" if a is None else str(a), + "" if b is None else str(b), + ] + try: + proc = subprocess.Popen( # nosec B603 + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + _windows_assign_job_memory_limit(proc) + try: + stdout, stderr = proc.communicate(timeout=60) + except subprocess.TimeoutExpired: + proc.kill() + proc.communicate() + return {"error": "trace subprocess timed out (60s)", "events": []} + except OSError as exc: + return {"error": f"trace subprocess failed to start: {exc}", "events": []} + + stdout = (stdout or "").strip() + if not stdout: + err = (stderr or "").strip() or f"exit {proc.returncode}" + return {"error": f"trace subprocess produced no output: {err}", "events": []} + + # Worker prints one JSON object on the last non-empty line. + last = stdout.splitlines()[-1] + try: + data = json.loads(last) + except json.JSONDecodeError: + return { + "error": "trace subprocess returned invalid JSON", + "events": [], + "stderr": (stderr or "")[:500], + } + if not isinstance(data, dict): + return {"error": "trace subprocess returned non-object JSON", "events": []} + return data + + +def analyze_errors_static(pyfile: str) -> dict[str, Any]: + src = open(pyfile, encoding="utf-8", errors="ignore").read() tree = ast.parse(src) - raises: List[Dict[str, Any]] = [] - try_catches: List[Dict[str, Any]] = [] + raises: list[dict[str, Any]] = [] + try_catches: list[dict[str, Any]] = [] class V(ast.NodeVisitor): def visit_Raise(self, node: ast.Raise): @@ -60,10 +370,3 @@ def visit_Try(self, node: ast.Try): V().visit(tree) return {"raises": raises, "catches": try_catches} - - -def _coerce(x): - try: - return int(x) - except Exception: - return x diff --git a/cli/ucli/visual/delta.py b/cli/ucli/visual/delta.py index 222997f..a746fe3 100644 --- a/cli/ucli/visual/delta.py +++ b/cli/ucli/visual/delta.py @@ -1,11 +1,16 @@ -import json, html +import html +import json + + +def _nodes(m): + return set(m.get("functions", {}).keys()) -def _nodes(m): return set(m.get('functions', {}).keys()) def lens_delta_svg(old_path: str, new_path: str) -> str: - old = json.load(open(old_path, 'r', encoding='utf-8')) - new = json.load(open(new_path, 'r', encoding='utf-8')) - old_nodes = _nodes(old); new_nodes = _nodes(new) + old = json.load(open(old_path, encoding="utf-8")) + new = json.load(open(new_path, encoding="utf-8")) + old_nodes = _nodes(old) + new_nodes = _nodes(new) added = new_nodes - old_nodes removed = old_nodes - new_nodes common = new_nodes & old_nodes @@ -18,10 +23,13 @@ def lens_delta_svg(old_path: str, new_path: str) -> str: svg.append(f'Lens delta') y += 20 for q in sorted(common): - svg.append(f'{html.escape(q)}'); y += 18 + svg.append(f'{html.escape(q)}') + y += 18 for q in sorted(added): - svg.append(f'+ {html.escape(q)}'); y += 18 + svg.append(f'+ {html.escape(q)}') + y += 18 for q in sorted(removed): - svg.append(f'- {html.escape(q)}'); y += 18 + svg.append(f'- {html.escape(q)}') + y += 18 svg.append("") return "\n".join(svg) diff --git a/docs/SECURITY.md b/docs/SECURITY.md index c009e6c..b0fc744 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -17,3 +17,44 @@ We will acknowledge reports within 72 hours and provide a timeline for remediati This is an active repository; we support the latest `main` branch and the most recent tagged release. Security fixes may be backported at the maintainers’ discretion. The Python package targets the interpreter range in `requires-python` (see the root `pyproject.toml`). Prefer installing with pinned dependencies (`uv.lock` / `uv sync --frozen`) or the exported `requirements.txt` when deploying. + +## Bandit baseline (CLI) + +CI runs Bandit fail-closed: any **HIGH** finding fails the job, and any finding not present in the committed `.bandit-baseline.json` also fails (new LOW/MEDIUM drift). + +### Intentional `subprocess` (B404 / B603) + +The CLI uses `subprocess` in fixed-argv, no-shell helpers: + +| Location | Why | +| --- | --- | +| `cli/ucli/commands/tour_ops.py` | `tour_run` fixture execution | +| `cli/ucli/commands/doctor_ops.py` | Doctor environment probes (`node -v`) | +| `cli/ucli/commands/demo_ops.py` | Local demo HTTP server lifecycle | +| `cli/ucli/trace/pytrace.py` | Isolated trace worker (never `exec` in-process) | + +Each of those imports is marked `# nosec B404` with an inline rationale. Call sites that spawn processes use `# nosec B603` where Bandit flags the invoke. **Do not** silence B404/B603 for shell=`True`, user-controlled argv, or new call sites without review. + +The committed `.bandit-baseline.json` is refreshed to **zero residual findings** when intentional `subprocess` imports are `# nosec B404`'d and call sites use `# nosec B603` where needed. Re-run `bandit -r cli/ -f json -o .bandit-baseline.json` after accepting any *new* residual finding — never to hide HIGH severity or unreviewed drift. Fail-closed gates (`-lll` HIGH + `-b` baseline) stay mandatory. + +## Runtime trace sandbox (`u trace module`) + +Tracing **must** execute target Python to collect call events. Mitigations: + +| Control | Behavior | +| --- | --- | +| Process isolation | Worker runs in a subprocess; the CLI process never `exec_module`s the target | +| Path allowlist | Target must resolve under cwd or `UF_TRACE_ALLOW_ROOTS`; non-`.py` refused | +| Argv denylist | `func` must be a simple identifier; dunders and `eval`/`exec`/`__import__`/`compile`/`open`/… are refused | +| POSIX limits | Soft `RLIMIT_CPU` / `RLIMIT_AS` in the worker when `resource` exists | +| Windows Job Object | Best-effort process memory cap via Job Object (ctypes); failure is non-fatal | +| Timeout | 60s hard kill | + +### Residual risk + +- Target code still runs with the privileges of the user invoking `u trace`. A malicious `.py` under an allowlisted root can perform any action the OS user can (network, filesystem outside the job memory limit, etc.). +- Windows Job Object assignment is best-effort; if ctypes/API calls fail, only the timeout remains as a hard stop (no POSIX `RLIMIT_*`). +- Denylist blocks *names* passed to `getattr(module, name)` — it does not inspect what the callable body does. +- Prefer `u trace errors` (AST-only) when execution is not required. + +Do not treat trace as a general-purpose sandbox for untrusted code. diff --git a/docs/usage.md b/docs/usage.md index c159015..2bc96a5 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -50,10 +50,45 @@ Checks Python and Node availability, grpc_tools, open ports, VS Code, and repo w 1) Scan the repository ```bash u scan . -o maps/repo.json -``` +# Optional: limit adapters (python, javascript, go, java, rust, csharp; aliases js/ts/typescript/golang/cs/c#) +u scan . --lang python -o maps/repo.json +u scan examples/js_toy --lang javascript -o maps/js.json +u scan examples/go_toy --lang go -o maps/go.json +u scan examples/java_toy --lang java -o maps/java.json +u scan examples/rust_toy --lang rust -o maps/rust.json +u scan examples/csharp_toy --lang csharp -o maps/csharp.json +``` + +`u scan` dispatches registered language adapters by file extension. Python uses +the AST analyzer. JavaScript/TypeScript (`.js`/`.mjs`/`.cjs`/`.ts`/`.tsx`/`.jsx`) +prefer a **Node + TypeScript compiler API** path (`javascript-ast` / +`typescript-ast`, complexity `ast-cyclomatic`) when Node 18+ and +`npm install` in `cli/ucli/analyzers/js_ast` are available; otherwise they +**degrade** to the regex adapter (`*-best-effort`, `keyword-heuristic`). +Force with `UF_JS_ANALYZER=regex|ast|auto`. Relative imports may also resolve via +re-export barrels and nearest `package.json` `"exports"` subpaths (AST path). +Go (`.go`) prefers **`go/parser`** +via `go run` in `cli/ucli/analyzers/go_ast` (`go-ast`, `ast-cyclomatic`) when +Go 1.21+ is on PATH; otherwise `go-best-effort` regex. Force with +`UF_GO_ANALYZER=regex|ast|auto`. Java (`.java`) is **`javalang` AST** when installed (`java-ast`, +`ast-cyclomatic`; `pip install javalang` or `understand-first[analyzers]`); +otherwise **best-effort regex** (`java-best-effort`). Force with +`UF_JAVA_ANALYZER=regex|ast|auto`. Rust (`.rs`) prefers **`syn`** via +`cargo run` in `cli/ucli/analyzers/rust_ast` (`rust-ast`, `ast-cyclomatic`) when +cargo is on PATH; otherwise **best-effort regex** (`rust-best-effort`). Force with +`UF_RUST_ANALYZER=regex|ast|auto`. C# (`.cs`) prefers **Roslyn** via +`dotnet run` in `cli/ucli/analyzers/csharp_ast` (`csharp-ast`, `ast-cyclomatic`) when +a .NET SDK is on PATH (`dotnet --list-sdks` non-empty); otherwise **best-effort +regex** (`csharp-best-effort`). Force with `UF_CSHARP_ANALYZER=regex|ast|auto`. +JS, Go, Java, Rust, and C# are not Python-parity. Unsupported source extensions +(e.g. `.rb`) are counted as not analyzed — they are never fed to the Python +analyzer or emitted as empty fake maps. 2) Create a task lens from seeds (files, functions, or labels) ```bash u lens from-seeds --map maps/repo.json --seed examples/app/hot_path.py -o maps/lens.json +# Default seed match is substring (tok in qname). Use --exact for full qname / +# local name / simple_name equality only: +u lens from-seeds --map maps/repo.json --seed compute --exact -o maps/lens.json # or use a preset label (from .understand-first.yml) u lens preset bug --map maps/repo.json -o maps/lens.json ``` @@ -61,6 +96,21 @@ u lens preset bug --map maps/repo.json -o maps/lens.json ```bash u trace module examples/app/hot_path.py run_hot_path -o traces/tour.json ``` + +Trace targets must resolve under the current working directory, or under extra +roots listed in ``UF_TRACE_ALLOW_ROOTS`` (``os.pathsep``-separated absolute or +relative directories). Example (PowerShell): + +```powershell +$env:UF_TRACE_ALLOW_ROOTS = "D:\safe\traces;D:\other\allow" +u trace module D:\safe\traces\mod.py run -o traces/out.json +``` + +Callable names are restricted to simple identifiers; a denylist refuses +``eval`` / ``exec`` / ``__import__`` / ``open`` and similar. See +[SECURITY.md](SECURITY.md) for residual risk (subprocess still executes target +code; Windows Job Object memory limits are best-effort). + 4) Merge runtime trace into the lens and rank by error proximity ```bash u lens merge-trace maps/lens.json traces/tour.json -o maps/lens_merged.json @@ -69,8 +119,14 @@ u lens merge-trace maps/lens.json traces/tour.json -o maps/lens_merged.json ```bash u tour maps/lens_merged.json -o tours/local.md ``` -6) Optional gate: verify walkthrough milestones +6) Optional gates ```bash +# CI fixture smoke (preferred PR gate — see tour-must-pass.yml): +u tour_run maps/lens_merged.json -f fixtures + +# IDE walkthrough milestones (requires .uf-progress.json from the VS Code panel, +# or produce one via fixture): +u tour_gate --fixture-lens maps/lens_merged.json u tour_gate --progress .uf-progress.json ``` @@ -82,7 +138,7 @@ u lens ingest-jira path/to/jira.json > seeds.json ``` ## Contracts -Generate and manage contracts from interface definitions with formal verification support. +Generate and manage contracts from interface definitions. Lean output is scaffold-only (presence checks), not compiled formal verification. ### Basic workflow ```bash @@ -95,10 +151,10 @@ u contracts from-proto examples/apis/orders.proto -o contracts/contracts_from_pr # 3. Compose multiple contract sources into a single file u contracts compose -i contracts/contracts_from_openapi.yaml -i contracts/contracts_from_proto.yaml -o contracts/contracts.yaml -# 4. Generate Lean formal verification stubs +# 4. Generate Lean scaffold stubs (Prop := True / by trivial — not compiled) u contracts lean-stubs contracts/contracts.yaml -o contracts/lean/ -# 5. Verify Lean coverage for all contract functions +# 5. Presence-only check that scaffold theorem stubs exist (does not compile Lean) u contracts verify-lean contracts/contracts.yaml -l contracts/lean # 6. Generate property test stubs @@ -108,17 +164,17 @@ u contracts stub-tests contracts/contracts.yaml -o tests/test_contracts.py ### Contract composition The `compose` command merges multiple contract YAML files, deduplicating modules and functions while preserving order. This is useful when you have contracts from different sources (OpenAPI, protobuf, manual definitions) that need to be unified. -### Lean verification -The `lean-stubs` command generates one Lean file per module containing: -- `invariant_{module}__{function}` definitions for each contract function -- `theorem` stubs for formal verification of contract properties +### Lean scaffold (presence-only) +The `lean-stubs` command generates one Lean file per module containing scaffold-only: +- `invariant_{module}__{function} : Prop := True` +- `theorem ... := by trivial` -The `verify-lean` command checks that every function in your contracts has a corresponding Lean theorem stub, helping ensure complete formal verification coverage. +These are **not** real proofs and are not compiled by Understand-First. The `verify-lean` command checks file presence of those theorem names (`mode: presence-only`); it does not run `lake build` or the Lean compiler. ### Contract structure Contracts are organized by modules (e.g., `ROUTE::pets`, `PROTO::orders`) with functions containing: - Request/response schemas as compact JSON metadata -- Pre/post conditions for formal verification +- Pre/post condition fields (documentation; Lean scaffold does not prove them) - Side effect annotations ## Visualization @@ -180,6 +236,7 @@ uv run pytest -q ``` ### PR gates -- Run `u scan`, `u lens from-seeds`, and `u tour` in CI to attach artifacts to PRs -- Use a PR gate that runs `u tour_gate` and fails the build if walkthrough milestones are not met -- The contracts verification step will fail if Lean stubs are missing for any contract function +- Fail-closed: `u scan`, `u lens from-seeds`, `u trace` / merge, `u tour`, then `u tour_run` (see `.github/workflows/tour-must-pass.yml`) +- Map delta: `u scan` base/head + `u diff --old/--new` (see `understand-first-pr-analysis.yml`) +- `u tour_gate` is for IDE milestone progress (`.uf-progress.json`); do not treat it as a substitute for `tour_run` in CI +- The contracts `verify-lean` step fails if scaffold theorem stubs are missing for any contract function (presence-only; does not compile Lean) diff --git a/pyproject.toml b/pyproject.toml index d36b3ef..eb97e40 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,9 @@ examples = [ "click>=8.1.7", "colorama>=0.4.6", ] +analyzers = [ + "javalang>=0.13.0", +] [project.scripts] u = "cli.ucli.main:app" @@ -78,6 +81,22 @@ Issues = "https://github.com/sentinelops-ci/understand-first/issues" [tool.setuptools.packages.find] where = ["."] +include = ["cli*"] +exclude = [ + "tests*", + "examples*", + "maps*", + "web_demo*", + "build*", + "dist*", + "dashboard*", + "commenter*", + "instrumentation*", + "scripts*", + "fixtures*", + "contracts*", + "templates*", +] [tool.setuptools.package-data] "cli.ucli" = ["py.typed"] @@ -94,6 +113,13 @@ src = ["cli", "tests"] select = ["E", "F", "I", "B", "UP"] ignore = ["E501"] +# Narrow, documented exceptions only: +# - E402: main.py must bootstrap sys.path / stdout before importing ucli.* and Typer. +# - B008: Typer's Option()/Argument() defaults are the supported CLI idiom. +[tool.ruff.lint.per-file-ignores] +"cli/ucli/main.py" = ["E402", "B008"] +"cli/ucli/commands/typer_groups.py" = ["B008"] + [tool.ruff.format] quote-style = "double" @@ -105,3 +131,4 @@ typeCheckingMode = "basic" reportMissingImports = true reportMissingTypeStubs = false reportRedeclaration = "none" + diff --git a/pytest.ini b/pytest.ini index 91f0d1d..8061634 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,3 @@ [pytest] addopts = -q +asyncio_default_fixture_loop_scope = function diff --git a/schemas/SCHEMA_REFERENCE.md b/schemas/SCHEMA_REFERENCE.md index 15b52ea..369e853 100644 --- a/schemas/SCHEMA_REFERENCE.md +++ b/schemas/SCHEMA_REFERENCE.md @@ -1,5 +1,16 @@ # Understand-First JSON Schema Reference +> **Honesty note (Wave 3 / Wave 15):** Runtime config is `cli/ucli/config.py` `SCHEMA` only. +> Python maps emit integer McCabe `complexity`, `calls`/`callers`, and heuristic +> `side_effects` string tags. JavaScript/TypeScript maps (Wave 15) are +> **best-effort** regex extracted defs/calls with a keyword complexity heuristic and +> same-file call edges only — they do **not** claim Python AST parity, import-gated +> resolution, or McCabe. Unsupported languages appear under `unsupported` counts +> and are **not** analyzed (no empty fake function maps). Examples below that show +> `cognitive` / `halstead`, rich documentation fields, or full multi-language +> fidelity are **illustrative / aspirational** unless the field is produced by +> `u scan` today. + This document provides comprehensive documentation for all Understand-First JSON schemas, including usage examples, validation rules, and integration guidelines. ## Table of Contents @@ -78,6 +89,57 @@ validate(instance=map_data, schema=map_schema) - `modules`: Logical module structure and dependencies - `relationships`: Call graphs and function relationships +### Qualified names (Python maps) + +Map keys are **qualified names** of the form `rel/path:local_name`: + +| Local name form | Example key | Notes | +| --- | --- | --- | +| Top-level function | `pkg/mod:compute` | Backward-compatible bare def name after `:` | +| Class method | `pkg/mod:Foo.run` | Nested classes use `Outer.Inner.method` | +| Nested function | `pkg/mod:Foo.run.helper` | Nested under the enclosing local name | + +Each function object also includes `simple_name`: the bare basename (`helper` for `Foo.helper`, `compute` for `compute`). Use `simple_name` for short-name UX; prefer full qnames for exact identity. + +**Call resolution** (high confidence only — ambiguous names omit edges rather than invent them): + +- Bare calls resolve same-file first, then unique import aliases / unique globals +- `mod.fn` / `pkg.mod.fn` via import aliases when uniquely matched +- `self.method` uses enclosing class when known (`Class.method`); unbound `self` outside a class body is refused (no same-file basename guess) +- `from mod import Foo` then `Foo.method` / `obj = Foo(); obj.method` when `Foo` uniquely maps to a scanned class under `mod` (import-gated; ambiguous same-named classes in other modules do not invent edges) +- `import mod as m` then `m.fn` / `obj = m.Foo(); obj.method` when uniquely matched; `from . import Foo` resolves against package `__init__` when unambiguous +- `obj.method` when `obj` has a high-confidence type from a constructor (`obj = Foo()`), annotation (`obj: Foo`, `Optional[Foo]`, `Foo | None`), or **return-type propagation** (`obj = make()` where `make` uniquely returns a concrete class via `-> Foo`, `-> models.Foo`, `from mod import Foo as F` then `-> F` / `return F()`, a single `return Foo(...)`, or `return alias` bound to that class). **Multi-hop factory chains** (`outer` returns `inner()` / `return x` after `x = inner()`, up to 3 hops) adopt `inner`'s concrete type when every hop uniquely resolves and agrees; cycles and ambiguous callees omit. Imported return types gate `Class.method` to the import module (so same-named classes in other files do not invent edges). Multi-return / union / unimported ambiguous names omit edges. When origin is only the factory file (same-module class), resolve prefers that file then unique global. Relative class imports (`from ..pkg.models import Foo as F`) are uniqueness-gated the same way. +- **Honesty rule (Wave 14):** untyped `obj.method` is **refused** even when a same-file method basename is unique. Prefer no edge over a guessed edge. Typed / `self` / import-gated paths above are unchanged. +- High-confidence `return_type` (concrete class name, import aliases canonicalized) is emitted on function entries when inferred; it is absent when ambiguous. Reports list inferred return types; the dashboard counts functions that expose `return_type` when present in the uploaded map. + +### Multi-language map emission (Wave 15–24) + +`u scan` merges outputs from registered adapters (`cli/ucli/analyzers/registry.py`): + +| Language | Fidelity | What is emitted | Limits | +| --- | --- | --- | --- | +| Python (`.py`) | AST | McCabe `complexity`, high-confidence `calls`/`callers`, heuristic `side_effects` | Same honesty rules as above | +| JavaScript / TypeScript (`.js`, `.mjs`, `.cjs`, `.ts`, `.tsx`, `.jsx`) | **AST preferred** (`javascript-ast` / `typescript-ast`) via Node + TypeScript compiler API; **regex fallback** (`*-best-effort`) if Node or `cli/ucli/analyzers/js_ast` deps missing | Defs + calls; `ast-cyclomatic` or keyword-heuristic; same-file unique edges; AST path may qualify **relative** `./` / `../` imports (incl. re-export barrels ≤2 hops) and nearest **`package.json` `"exports"`** subpaths when uniquely resolved | Not Python parity: no typed `obj.method`, no bare npm package invent, no typechecking. Override with `UF_JS_ANALYZER=regex\|ast\|auto` | +| Go (`.go`) | **AST preferred** (`go-ast`) via `go run` + `go/parser`; **regex fallback** (`go-best-effort`) if Go toolchain missing | Defs + methods; `ast-cyclomatic` or keyword-heuristic; same-file unique edges; AST path may qualify **same-package** unique names across scanned files | Not Python parity: no cross-package invent, no typed method resolve. Override with `UF_GO_ANALYZER=regex\|ast\|auto` | +| Java (`.java`) | **AST preferred** (`java-ast`) via optional ``javalang``; **regex fallback** (`java-best-effort`) | Class/method defs; `ast-cyclomatic` or keyword-heuristic; same-file + same-package unique edges | Not typechecked; no cross-package invent. Install `javalang` / `understand-first[analyzers]`. Override with `UF_JAVA_ANALYZER=regex\|ast\|auto` | +| Rust (`.rs`) | **AST preferred** (`rust-ast`) via `cargo run` + `syn`; **regex fallback** (`rust-best-effort`) if cargo missing | `fn` / `impl` methods; `ast-cyclomatic` or keyword-heuristic; same-file unique edges only | Not rustc typecheck; no `use`/crate-path invent. Override with `UF_RUST_ANALYZER=regex\|ast\|auto` | +| C# (`.cs`) | **AST preferred** (`csharp-ast`) via `dotnet run` + Roslyn; **regex fallback** (`csharp-best-effort`) if .NET SDK missing | Class/method defs; `ast-cyclomatic` or keyword-heuristic; same-file + same-namespace unique edges | Not typechecked / not MSBuild; no cross-namespace / `using` invent. Requires SDK (`dotnet --list-sdks` non-empty). Override with `UF_CSHARP_ANALYZER=regex\|ast\|auto` | +| Other source (`.rb`, `.php`, …) | none | Counted under `unsupported` | **Not analyzed** — never fed to Python as empty maps | + +**JS/TS `ast-cyclomatic` formula:** `1` plus counts of `If` / `For` / `ForIn` / `ForOf` / `While` / `DoWhile` / `CaseClause` / `CatchClause` / conditional `?:` / `&&` / `||` in the function body; nested function bodies are excluded (counted on their own entries). + +**Go `ast-cyclomatic` formula:** `1` plus `IfStmt` / `ForStmt` / `RangeStmt` / `CaseClause` / `&&` / `||` extras; nested `FuncLit` bodies excluded. + +**C# `ast-cyclomatic` formula:** `1` plus `If` / `For` / `ForEach` / `While` / `Do` / `Case` / `Catch` / conditional `?:` / `&&` / `||`; nested local function / lambda bodies excluded. + +Optional map fields from the dispatcher: `languages_analyzed`, `analyzer_fidelity`, `analyzer_notes`, `unsupported`, `files_analyzed`, `complexity_kind` (`mccabe` | `ast-cyclomatic` | `keyword-heuristic` | `mixed` | `none`), and (for single-language maps) `analyzer` such as `javascript-ast`, `go-ast`, `java-best-effort`, `csharp-ast`, or `go-best-effort`. Per-function entries carry `analyzer`, `complexity_kind`, and per-file `language`. Average complexity in the CLI summary is computed only over analyzed functions (never invented for unsupported languages). Filter with `u scan --lang python` / `--lang javascript` / `--lang typescript` / `--lang go` / `--lang java` / `--lang rust` / `--lang csharp` (aliases: `cs`, `c#`; typescript selects the JS adapter; TS-only trees label `language: typescript`). + +**Wave 16–24 consumer honesty:** `u lens` / `u tour` work on JS/Go/Java/Rust/C# maps as reading-plan artifacts and copy fidelity metadata onto the lens. `u scan` / `u report` / `u diff` label whether complexity is McCabe, ast-cyclomatic, or keyword-heuristic (mixed totals are not uniform McCabe). `u tour_run` / `u trace` **refuse** non-Python lenses (Python runtime only). Install JS AST deps with `npm install` in `cli/ucli/analyzers/js_ast`; Go AST needs Go 1.21+ on PATH (`cli/ucli/analyzers/go_ast`); Java AST needs `javalang`; Rust AST needs `cargo` (`cli/ucli/analyzers/rust_ast`); C# AST needs a .NET SDK 8+ (`cli/ucli/analyzers/csharp_ast`). + +### Lens seeds: substring vs exact + +By default, `u lens from-seeds --seed TOKEN` uses **substring** matching (`TOKEN in qname`). That is convenient but broad (`run` also matches `runner`). Pass `--exact` to match only full qname, local name after `:`, or `simple_name` equality. + ## Lens Schema **File**: `lens-schema.json` diff --git a/schemas/config-schema.json b/schemas/config-schema.json index 7068af8..03ee210 100644 --- a/schemas/config-schema.json +++ b/schemas/config-schema.json @@ -2,7 +2,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "$id": "https://understand-first.dev/schemas/config-schema.json", "title": "Understand-First Configuration", - "description": "Schema for Understand-First configuration files", + "description": "Aspirational / documentation schema for configuration files. Runtime source of truth is cli/ucli/config.py SCHEMA (keys: seeds, hops, contracts_paths, glossary_path, seeds_for, metrics.enabled only). Keys such as analysis.complexity.metrics cognitive/halstead, multi-language repository.languages, and most visualization/ci knobs are NOT read by the analyzer today — treat them as future/docs unless present in runtime SCHEMA.", "type": "object", "required": ["version"], "properties": { @@ -58,11 +58,12 @@ "threshold": {"type": "integer", "default": 10}, "metrics": { "type": "array", + "description": "UNIMPLEMENTED except cyclomatic via Python McCabe in u scan. cognitive/halstead/maintainability are not computed.", "items": { "type": "string", "enum": ["cyclomatic", "cognitive", "halstead", "maintainability"] }, - "default": ["cyclomatic", "cognitive"] + "default": ["cyclomatic"] }, "ignore_patterns": { "type": "array", @@ -72,6 +73,7 @@ }, "side_effects": { "type": "object", + "description": "Config knobs unused at runtime. Python scan always emits heuristic side_effects tags (see python_analyzer). detect_database and similar toggles are NOT implemented.", "properties": { "enabled": {"type": "boolean", "default": true}, "detect_io": {"type": "boolean", "default": true}, diff --git a/schemas/lens-schema.json b/schemas/lens-schema.json index 20146c0..1c6997c 100644 --- a/schemas/lens-schema.json +++ b/schemas/lens-schema.json @@ -27,11 +27,16 @@ }, "seeds": { "type": "array", - "description": "Seed functions or files that define the lens scope", + "description": "Seed functions or files that define the lens scope. Default matching is substring (tok in qname); see exact_match.", "items": { "type": "string" } }, + "exact_match": { + "type": "boolean", + "description": "When true, seeds match full qname, local name after ':', or simple_name only (not substring)", + "default": false + }, "created_at": { "type": "string", "format": "date-time", diff --git a/schemas/map-schema.json b/schemas/map-schema.json index 0d5caf5..81057e4 100644 --- a/schemas/map-schema.json +++ b/schemas/map-schema.json @@ -2,7 +2,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "$id": "https://understand-first.dev/schemas/map-schema.json", "title": "Understand-First Repository Map", - "description": "Schema for repository maps generated by `u scan` command", + "description": "Schema for repository maps generated by `u scan` command. Python maps are AST-backed; JavaScript/TypeScript maps prefer a Node TypeScript-compiler AST path (Wave 17) and fall back to best-effort regex when Node/deps are missing. Unsupported languages are omitted from `functions` and may appear only as inventory/counts in dispatcher metadata.", "type": "object", "required": ["version", "repository", "functions", "files"], "properties": { @@ -37,7 +37,7 @@ "language": { "type": "string", "description": "Primary programming language", - "enum": ["python", "javascript", "typescript", "go", "rust", "java", "csharp", "mixed"] + "enum": ["python", "javascript", "typescript", "go", "rust", "java", "csharp", "mixed", "none"] }, "total_files": { "type": "integer", @@ -49,6 +49,40 @@ } } }, + "languages_analyzed": { + "type": "array", + "description": "Adapter language ids that contributed functions (Wave 15 dispatcher)", + "items": { "type": "string" } + }, + "analyzer_fidelity": { + "type": "object", + "description": "Per-language fidelity label (e.g. ast, best-effort)", + "additionalProperties": { "type": "string" } + }, + "analyzer_notes": { + "type": "object", + "description": "Per-language honesty notes from adapters", + "additionalProperties": { "type": "string" } + }, + "unsupported": { + "type": "object", + "description": "Counts of discovered source files with no adapter (not analyzed; never faked)", + "additionalProperties": { "type": "integer", "minimum": 0 } + }, + "files_analyzed": { + "type": "integer", + "description": "Count of files handed to registered adapters", + "minimum": 0 + }, + "analyzer": { + "type": "string", + "description": "Primary analyzer id when a single adapter produced the map (e.g. javascript-ast, javascript-best-effort)" + }, + "complexity_kind": { + "type": "string", + "description": "How complexity integers should be interpreted", + "enum": ["mccabe", "ast-cyclomatic", "keyword-heuristic", "mixed", "none"] + }, "functions": { "type": "object", "description": "Map of function qualified names to function metadata", @@ -107,30 +141,82 @@ }, "return_type": { "type": "string", - "description": "Function return type" + "description": "High-confidence concrete class return type when uniquely inferred from a return annotation or a single return Class(...)/alias (import aliases and mod.Class forms canonicalize to the class name and gate cross-file obj.method); omitted when multi-return or ambiguous" }, "calls": { "type": "array", - "description": "Functions called by this function", + "description": "Callees: file-local or high-confidence cross-file names are qualified (path:func or path:Class.method), including from-import class/typed obj.method and return-propagated factory obj.method when uniquely gated; otherwise bare short names or unresolved dotted tokens (mod.fn, self.method, obj.method)", "items": { "type": "string" } }, "callers": { "type": "array", - "description": "Functions that call this function", + "description": "Qualified callers (path:func or path:Class.method) that call this function", "items": { "type": "string" } }, "side_effects": { "type": "array", - "description": "Side effects performed by this function", + "description": "Heuristic AST side-effect tags from Python scan (not proven purity). See python_analyzer heuristics.", "items": { "type": "string", - "enum": ["file_read", "file_write", "network_request", "database_query", "cache_operation", "logging", "exception_raised"] + "enum": [ + "writes_global", + "writes_attribute", + "io_open", + "io_print", + "io_write", + "io_read", + "network_call", + "logging" + ] } }, + "has_docstring": { + "type": "boolean", + "description": "True when ast.get_docstring finds a docstring on the function" + }, + "has_return_annotation": { + "type": "boolean", + "description": "True when the function has a return annotation" + }, + "has_type_hints": { + "type": "boolean", + "description": "True when any param or return annotation is present" + }, + "typed_params": { + "type": "integer", + "description": "Count of parameters with annotations", + "minimum": 0 + }, + "total_params": { + "type": "integer", + "description": "Total parameter count (args + kwonly + vararg + kwarg)", + "minimum": 0 + }, + "fully_typed_params": { + "type": "boolean", + "description": "True when every parameter has an annotation (or there are none)" + }, + "simple_name": { + "type": "string", + "description": "Bare def/method basename (e.g. helper for Class.helper). Use for short-name UX; map keys use Class.method local names." + }, + "analyzer": { + "type": "string", + "description": "Producing analyzer id (e.g. javascript-best-effort). Python AST maps may omit this." + }, + "complexity_kind": { + "type": "string", + "description": "Interpretation of complexity for this function", + "enum": ["mccabe", "ast-cyclomatic", "keyword-heuristic"] + }, + "language": { + "type": "string", + "description": "Source language for this function entry when set by a multi-language adapter" + }, "imports": { "type": "array", "description": "Modules imported by this function's file", diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/ci/__init__.py b/scripts/ci/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py index c5b4549..28bfe4b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,8 @@ -import sys, pathlib +import pathlib +import sys ROOT = pathlib.Path(__file__).resolve().parents[1] -if str(ROOT) not in sys.path: - sys.path.insert(0, str(ROOT)) +CLI = ROOT / "cli" +for _p in (ROOT, CLI): + if str(_p) not in sys.path: + sys.path.insert(0, str(_p)) diff --git a/tests/test_cli_smoke.py b/tests/test_cli_smoke.py index b19cd5f..468b30e 100644 --- a/tests/test_cli_smoke.py +++ b/tests/test_cli_smoke.py @@ -1,9 +1,14 @@ -import os, subprocess, sys, time, json, pathlib +import os +import pathlib +import subprocess +import sys +import time ROOT = pathlib.Path(__file__).resolve().parents[1] + def run(cmd, cwd=None, check=True): - print('>', cmd) + print(">", cmd) env = os.environ.copy() env.setdefault("PYTHONUTF8", "1") env.setdefault("PYTHONIOENCODING", "utf-8") @@ -20,17 +25,22 @@ def run(cmd, cwd=None, check=True): env=env, ) + def test_scan_and_lens(): - run('u scan examples/python_toy -o maps/repo.json') - run('u lens from-seeds --map maps/repo.json -o maps/lens.json') + run("u scan examples/python_toy -o maps/repo.json") + run("u lens from-seeds --map maps/repo.json -o maps/lens.json") + def test_trace_and_tour(): # start HTTP server - p = subprocess.Popen([sys.executable, 'examples/servers/http_server.py'], cwd=ROOT) + p = subprocess.Popen([sys.executable, "examples/servers/http_server.py"], cwd=ROOT) try: time.sleep(1.5) - run('u trace module examples/app/hot_path.py run_hot_path -o traces/tour.json', check=False) - run('u lens merge-trace maps/lens.json traces/tour.json -o maps/lens_merged.json', check=False) - run('u tour maps/lens_merged.json -o tours/test.md', check=False) + run("u trace module examples/app/hot_path.py run_hot_path -o traces/tour.json", check=False) + run( + "u lens merge-trace maps/lens.json traces/tour.json -o maps/lens_merged.json", + check=False, + ) + run("u tour maps/lens_merged.json -o tours/test.md", check=False) finally: p.terminate() diff --git a/tests/test_config_and_openapi.py b/tests/test_config_and_openapi.py index 6ae3669..72841d2 100644 --- a/tests/test_config_and_openapi.py +++ b/tests/test_config_and_openapi.py @@ -1,4 +1,3 @@ -import json, os, tempfile, pathlib from cli.ucli.config import validate_config_dict from cli.ucli.contracts.contracts import from_openapi from cli.ucli.lens.lens import explain_node diff --git a/tests/test_contracts_and_boundaries.py b/tests/test_contracts_and_boundaries.py index 3fd6d55..e936adb 100644 --- a/tests/test_contracts_and_boundaries.py +++ b/tests/test_contracts_and_boundaries.py @@ -1,5 +1,5 @@ -from cli.ucli.contracts.contracts import from_openapi, from_proto, report_json from cli.ucli.boundaries.scan import scan_boundaries +from cli.ucli.contracts.contracts import from_openapi, from_proto def test_from_openapi_and_proto_and_report(): @@ -15,7 +15,8 @@ def test_from_openapi_and_proto_and_report(): def report_json_string(txt: str): # helper to feed string to report_json logic - import tempfile, os + import os + import tempfile fd, path = tempfile.mkstemp(suffix=".yaml") try: diff --git a/tests/test_error_tracer.py b/tests/test_error_tracer.py index 8395793..cb16a5a 100644 --- a/tests/test_error_tracer.py +++ b/tests/test_error_tracer.py @@ -1,5 +1,7 @@ +import os +import tempfile + from cli.ucli.trace.pytrace import analyze_errors_static -import tempfile, os def test_analyze_errors_static_finds_raises_and_catches(): diff --git a/tests/test_lang_strategies.py b/tests/test_lang_strategies.py index 37676a4..6b75e70 100644 --- a/tests/test_lang_strategies.py +++ b/tests/test_lang_strategies.py @@ -1,8 +1,8 @@ from cli.ucli.contracts.lang_strategies import ( - strategy_for_java_type, + infer_csharp_method_args, infer_java_method_args, strategy_for_csharp_type, - infer_csharp_method_args, + strategy_for_java_type, ) diff --git a/tests/test_remediation_wave.py b/tests/test_remediation_wave.py new file mode 100644 index 0000000..f82169a --- /dev/null +++ b/tests/test_remediation_wave.py @@ -0,0 +1,97 @@ +"""Focused regressions for Phase 0/1 remediation wave.""" + +from __future__ import annotations + +import pathlib +import zipfile + +from cli.ucli.analyzers.python_analyzer import _parse_file, build_python_map +from cli.ucli.config import filter_config_to_schema, validate_config_dict +from cli.ucli.contracts.contracts import lean_stubs + + +def test_lean_stubs_write_real_newlines(tmp_path: pathlib.Path): + contracts = tmp_path / "c.yaml" + contracts.write_text( + "module: demo\nfunctions:\n foo:\n pre: []\n post: []\n side_effects: []\n", + encoding="utf-8", + ) + out_dir = tmp_path / "lean" + count = lean_stubs(str(contracts), str(out_dir)) + assert count >= 1 + lean_files = list(out_dir.glob("*.lean")) + assert lean_files + text = lean_files[0].read_text(encoding="utf-8") + assert "\\n" not in text + assert "\n" in text + assert "namespace Contracts" in text + assert "Prop := True" in text + + +def test_verify_lean_is_presence_only(tmp_path: pathlib.Path): + from cli.ucli.contracts.contracts import verify_lean + + contracts = tmp_path / "c.yaml" + contracts.write_text( + "module: demo\nfunctions:\n foo:\n pre: []\n post: []\n side_effects: []\n", + encoding="utf-8", + ) + lean_dir = tmp_path / "lean" + lean_stubs(str(contracts), str(lean_dir)) + data = verify_lean(str(contracts), str(lean_dir)) + assert data["mode"] == "presence-only" + assert data["missing_invariants"] == [] + assert data["functions_total"] >= 1 + + +def test_async_function_def_parsed_like_sync(tmp_path: pathlib.Path): + src = tmp_path / "async_mod.py" + src.write_text( + "async def fetch():\n await helper()\n\nasync def helper():\n return 1\n", + encoding="utf-8", + ) + parsed = _parse_file(src) + assert "fetch" in parsed + assert "helper" in parsed["fetch"]["calls"] + + repo = build_python_map(tmp_path) + keys = "\n".join(repo.get("functions", {})) + assert ":fetch" in keys + assert ":helper" in keys + + +def test_config_schema_filter_drops_theater_keys(): + dirty = { + "hops": 2, + "seeds": ["main"], + "metrics": {"enabled": True, "weekly_reports": True}, + "exclude_patterns": ["**/tmp/**"], + "ci_integration": {"enabled": True}, + } + clean = filter_config_to_schema(dirty) + assert "exclude_patterns" not in clean + assert "ci_integration" not in clean + assert clean["metrics"] == {"enabled": True} + assert validate_config_dict(clean) == [] + + +def test_packaging_top_level_is_cli_only(): + """Wheel top-level must be cli (not tests/examples/maps/...).""" + root = pathlib.Path(__file__).resolve().parents[1] + pyproject = (root / "pyproject.toml").read_text(encoding="utf-8") + assert 'include = ["cli*"]' in pyproject + + # Prefer inspecting a freshly built wheel when present (CI/local verify). + dist = root / "dist_verify" + wheels = sorted(dist.glob("*.whl")) if dist.is_dir() else [] + if not wheels: + return + with zipfile.ZipFile(wheels[-1]) as zf: + top = zf.read(next(n for n in zf.namelist() if n.endswith("top_level.txt"))).decode() + assert top.strip().splitlines() == ["cli"] + foreign = [ + n + for n in zf.namelist() + if not n.startswith("cli/") and not n.startswith("understand_first") + ] + assert foreign == [] diff --git a/tests/test_report_and_visual.py b/tests/test_report_and_visual.py index 8cb661a..c6c77c6 100644 --- a/tests/test_report_and_visual.py +++ b/tests/test_report_and_visual.py @@ -1,7 +1,11 @@ +import json +import os +import pathlib +import tempfile + +from cli.ucli.analyzers.python_analyzer import build_python_map from cli.ucli.report.report import make_report_md from cli.ucli.visual.delta import lens_delta_svg -from cli.ucli.analyzers.python_analyzer import build_python_map -import json, tempfile, os, pathlib def test_make_report_md_lists_hotspots():