chore: New docs generation

.github/workflows/docs.yaml → .github/workflows/docs-update.yaml

@@ -1,11 +1,13 @@
+---
 name: Notify Documentation Update
 
 on:
   push:
     branches: [main]
     paths:
       - "docs/**"
-      - "scripts/make_docs.py"
+      - ".hooks/generate_docs.py"
+      - ".github/workflows/docs-update.yaml"
   workflow_dispatch:
 
 jobs:
@@ -28,4 +30,12 @@ jobs:
           token: ${{ steps.app-token.outputs.token }}
           repository: dreadnode/prod-docs
           event-type: code-update
-          client-payload: '{"repository": "${{ github.repository }}", "ref": "${{ github.ref }}", "sha": "${{ github.sha }}", "product": "strikes", "docs_dir": "docs", "module_dir": "dreadnode"}'
+          client-payload: |
+            {
+              "repository": "${{ github.repository }}",
+              "ref": "${{ github.ref }}",
+              "sha": "${{ github.sha }}",
+              "source_dir": "docs",
+              "target_dir": "strikes",
+              "nav_target": "Documentation/Strikes"
+            }

.github/workflows/publish.yaml

@@ -1,3 +1,4 @@
+---
 name: Build and Publish
 
 on:
@@ -46,4 +47,4 @@ jobs:
         run: poetry build
 
       - name: Publish to PyPI
-        uses: pypa/gh-action-pypi-publish@e9ccbe5a211ba3e8363f472cae362b56b104e796
+        uses: pypa/gh-action-pypi-publish@e9ccbe5a211ba3e8363f472cae362b56b104e796

.github/workflows/test.yaml

@@ -1,10 +1,11 @@
+---
 name: Tests
 
 on:
   push:
-    branches: [ main ]
+    branches: [main]
   pull_request:
-    branches: [ main ]
+    branches: [main]
 
 jobs:
   python:
@@ -52,4 +53,4 @@ jobs:
       run: poetry run mypy .
 
     - name: Test
-      run: poetry run pytest
+      run: poetry run pytest

.hooks/generate_docs.py

@@ -0,0 +1,222 @@
+import argparse  # noqa: INP001
+import re
+import typing as t
+from pathlib import Path
+
+from markdown import Markdown  # type: ignore[import-untyped]
+from markdownify import MarkdownConverter  # type: ignore[import-untyped]
+from markupsafe import Markup
+from mkdocstrings_handlers.python._internal.config import PythonConfig
+from mkdocstrings_handlers.python._internal.handler import (
+    PythonHandler,
+)
+
+# ruff: noqa: T201
+
+
+class CustomMarkdownConverter(MarkdownConverter):  # type: ignore[misc]
+    # Strip extra whitespace from code blocks
+    def convert_pre(self, el: t.Any, text: str, parent_tags: t.Any) -> t.Any:
+        return super().convert_pre(el, text.strip(), parent_tags)
+
+    # bold items with doc-section-title in a span class
+    def convert_span(self, el: t.Any, text: str, parent_tags: t.Any) -> t.Any:  # noqa: ARG002
+        if "doc-section-title" in el.get("class", []):
+            return f"**{text.strip()}**"
+        return text
+
+    # Remove the div wrapper for inline descriptions
+    def convert_div(self, el: t.Any, text: str, parent_tags: t.Any) -> t.Any:
+        if "doc-md-description" in el.get("class", []):
+            return text.strip()
+        return super().convert_div(el, text, parent_tags)
+
+    # Map mkdocstrings details classes to Mintlify callouts
+    def convert_details(self, el: t.Any, text: str, parent_tags: t.Any) -> t.Any:  # noqa: ARG002
+        classes = el.get("class", [])
+
+        # Handle source code details specially
+        if "quote" in classes:
+            summary = el.find("summary")
+            if summary:
+                file_path = summary.get_text().replace("Source code in ", "").strip()
+                content = text[text.find("```") :]
+                return f'\n<Accordion title="Source code in {file_path}" icon="code">\n{content}\n</Accordion>\n'
+
+        callout_map = {
+            "note": "Note",
+            "warning": "Warning",
+            "info": "Info",
+            "tip": "Tip",
+        }
+
+        callout_type = None
+        for cls in classes:
+            if cls in callout_map:
+                callout_type = callout_map[cls]
+                break
+
+        if not callout_type:
+            return text
+
+        content = text.strip()
+        if content.startswith(callout_type):
+            content = content[len(callout_type) :].strip()
+
+        return f"\n<{callout_type}>\n{content}\n</{callout_type}>\n"
+
+    def convert_table(self, el: t.Any, text: str, parent_tags: t.Any) -> t.Any:
+        # Check if this is a highlighttable (source code with line numbers)
+        if "highlighttable" in el.get("class", []):
+            code_cells = el.find_all("td", class_="code")
+            if code_cells:
+                code = code_cells[0].get_text()
+                code = code.strip()
+                code = code.replace("```", "~~~")
+                return f"\n```python\n{code}\n```\n"
+
+        return super().convert_table(el, text, parent_tags)
+
+
+class AutoDocGenerator:
+    def __init__(self, source_paths: list[str], theme: str = "material", **options: t.Any) -> None:
+        self.source_paths = source_paths
+        self.theme = theme
+        self.handler = PythonHandler(PythonConfig.from_data(), base_dir=Path.cwd())
+        self.options = options
+
+        self.handler._update_env(  # noqa: SLF001
+            Markdown(),
+            config={"mdx": ["toc"]},
+        )
+
+        md = Markdown(extensions=["fenced_code"])
+
+        def simple_convert_markdown(
+            text: str,
+            heading_level: int,
+            html_id: str = "",
+            **kwargs: t.Any,
+        ) -> t.Any:
+            return Markup(md.convert(text) if text else "")  # noqa: S704 # nosec
+
+        self.handler.env.filters["convert_markdown"] = simple_convert_markdown
+
+    def generate_docs_for_module(
+        self,
+        module_path: str,
+    ) -> str:
+        options = self.handler.get_options(
+            {
+                "docstring_section_style": "list",
+                "merge_init_into_class": True,
+                "show_signature_annotations": True,
+                "separate_signature": True,
+                "show_source": True,
+                "show_labels": False,
+                "show_bases": False,
+                **self.options,
+            },
+        )
+
+        module_data = self.handler.collect(module_path, options)
+        html = self.handler.render(module_data, options)
+
+        return str(
+            CustomMarkdownConverter(
+                code_language="python",
+            ).convert(html),
+        )
+
+    def process_mdx_file(self, file_path: Path) -> bool:
+        content = file_path.read_text(encoding="utf-8")
+        original_content = content
+
+        # Find the header comment block
+        header_match = re.search(
+            r"\{\s*/\*\s*((?:::.*?\n?)*)\s*\*/\s*\}",
+            content,
+            re.MULTILINE | re.DOTALL,
+        )
+
+        if not header_match:
+            return False
+
+        header = header_match.group(0)
+        module_lines = header_match.group(1).strip().split("\n")
+
+        # Generate content for each module
+        markdown_blocks = []
+        for line in module_lines:
+            if line.startswith(":::"):
+                module_path = line.strip()[3:].strip()
+                if module_path:
+                    markdown = self.generate_docs_for_module(module_path)
+                    markdown_blocks.append(markdown)
+
+        keep_end = content.find(header) + len(header)
+        new_content = content[:keep_end] + "\n\n" + "\n".join(markdown_blocks)
+
+        # Write back if changed
+        if new_content != original_content:
+            file_path.write_text(new_content, encoding="utf-8")
+            print(f"[+] Updated: {file_path}")
+            return True
+
+        return False
+
+    def process_directory(self, directory: Path, pattern: str = "**/*.mdx") -> int:
+        if not directory.exists():
+            print(f"[!] Directory does not exist: {directory}")
+            return 0
+
+        files_processed = 0
+        files_modified = 0
+
+        for mdx_file in directory.glob(pattern):
+            if mdx_file.is_file():
+                files_processed += 1
+                if self.process_mdx_file(mdx_file):
+                    files_modified += 1
+
+        return files_modified
+
+
+def main() -> None:
+    """Main entry point for the script."""
+
+    parser = argparse.ArgumentParser(description="Generate auto-docs for MDX files")
+    parser.add_argument("--directory", help="Directory containing MDX files", default="docs")
+    parser.add_argument("--pattern", default="**/*.mdx", help="File pattern to match")
+    parser.add_argument(
+        "--source-paths",
+        nargs="+",
+        default=["dreadnode"],
+        help="Python source paths for module discovery",
+    )
+    parser.add_argument(
+        "--show-if-no-docstring",
+        type=bool,
+        default=False,
+        help="Show module/class/function even if no docstring is present",
+    )
+    parser.add_argument("--theme", default="material", help="Theme to use for rendering")
+
+    args = parser.parse_args()
+
+    # Create generator
+    generator = AutoDocGenerator(
+        source_paths=args.source_paths,
+        theme=args.theme,
+        show_if_no_docstring=args.show_if_no_docstring,
+    )
+
+    # Process directory
+    directory = Path(args.directory)
+    modified_count = generator.process_directory(directory, args.pattern)
+
+    print(f"\n[+] Auto-doc generation complete. {modified_count} files were updated.")
+
+
+if __name__ == "__main__":
+    main()

.hooks/typing_and_linting.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -e
+
+poetry run mypy .
+poetry run ruff check .
+poetry run ruff format --check .

.pre-commit-config.yaml

@@ -14,6 +14,7 @@ repos:
       - id: check-yaml
       - id: detect-private-key
       - id: end-of-file-fixer
+        exclude: ^docs/
       - id: trailing-whitespace
 
   - repo: https://github.com/rhysd/actionlint
@@ -31,7 +32,7 @@ repos:
     rev: v2.4.1
     hooks:
       - id: codespell
-        entry: codespell -q 3 -f --skip=".git,.github,README.md" --ignore-words-list="astroid,braket,te"
+        entry: codespell -q 3 -f --skip=".git,.github,README.md" -L astroid,braket,te,ROUGE
 
   # Python code security
   - repo: https://github.com/PyCQA/bandit
@@ -57,22 +58,6 @@ repos:
       - id: nbstripout
         args: [--keep-id]
 
-  # - repo: https://github.com/astral-sh/ruff-pre-commit
-  #   rev: v0.11.7
-  #   hooks:
-  #     - id: ruff
-  #       args: [--fix]
-  #     - id: ruff-format
-
-  # - repo: https://github.com/pre-commit/mirrors-mypy
-  #   rev: v1.15.0
-  #   hooks:
-  #     - id: mypy
-  #       additional_dependencies:
-  #         - "types-PyYAML"
-  #         - "types-requests"
-  #         - "types-setuptools"
-
   - repo: local
     hooks:
       # Ensure our GH actions are pinned to a specific hash
@@ -82,8 +67,33 @@ repos:
         language: python
         files: \.github/.*\.yml$
 
+      # Format JSON and YAML files
       - id: prettier
         name: Run prettier
         entry: .hooks/prettier.sh
         language: script
         types: [json, yaml]
+
+      # Post-merge hook to refresh dependencies
+      - id: refresh-dependencies
+        name: Refresh Dependencies
+        entry: .hooks/refresh_dependencies.sh
+        language: script
+        stages: [post-merge]
+        always_run: true
+
+      # Pre-push hook to run typing and linting
+      - id: typing-and-linting
+        name: Typing and Linting
+        entry: .hooks/typing_and_linting.sh
+        language: script
+        stages: [pre-push]
+        always_run: true
+
+      # Generate documentation
+      - id: generate-docs
+        name: Generate docs
+        entry: poetry run python .hooks/generate_docs.py
+        language: system
+        pass_filenames: false
+        always_run: true

.vscode/extensions.json

@@ -7,4 +7,4 @@
         "ms-python.mypy-type-checker",
         "tamasfe.even-better-toml"
     ]
-}
+}

.vscode/settings.json

@@ -12,4 +12,4 @@
     ],
     "python.testing.unittestEnabled": false,
     "python.testing.pytestEnabled": true
-}
+}

README.md

@@ -97,4 +97,4 @@ Read through our **[introduction guide](https://docs.dreadnode.io/strikes/intro)
 
 ## Examples
 
-Check out **[dreadnode/example-agents](https://github.com/dreadnode/example-agents)** to find your favorite use case. 
+Check out **[dreadnode/example-agents](https://github.com/dreadnode/example-agents)** to find your favorite use case.