Nrichers/sc 15354/workflows fix notebook execution workflows

.github/workflows/deploy-docs-prod.yaml

@@ -117,7 +117,7 @@ jobs:
         run: aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID_PROD }} && aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY_PROD }} && aws configure set default.region us-east-1
 
       - name: Deploy docs prod site
-        run: aws s3 sync site/_site s3://validmind-docs-prod/site --delete --exclude "installation/omnibus/*" --exclude "installation/helm-repo/*" --exclude "llm/*" && aws cloudfront create-invalidation --distribution-id E2BGG3USKQTR9W --paths "/*" --no-cli-pager
+        run: aws s3 sync site/_site s3://validmind-docs-prod/site --delete --exclude "installation/omnibus/*" --exclude "installation/helm-repo/*" --exclude "notebooks/EXECUTED/*" --exclude "llm/*" && aws cloudfront create-invalidation --distribution-id E2BGG3USKQTR9W --paths "/*" --no-cli-pager
 
       # Release headroom and shrink before final lightweight steps & post-job
       - name: Release reserve & shrink

.github/workflows/deploy-docs-staging.yaml

@@ -120,7 +120,7 @@ jobs:
         run: aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID_STAGING }} && aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY_STAGING }} && aws configure set default.region us-west-2
 
       - name: Deploy docs staging site
-        run: aws s3 sync site/_site s3://validmind-docs-staging/site --delete --exclude "installation/helm-repo/*" --exclude "pr_previews/*" --exclude "llm/*" && aws cloudfront create-invalidation --distribution-id ESWVTZYFL873V --paths "/*" --no-cli-pager
+        run: aws s3 sync site/_site s3://validmind-docs-staging/site --delete --exclude "installation/helm-repo/*" --exclude "pr_previews/*" --exclude "notebooks/EXECUTED/*" --exclude "llm/*" && aws cloudfront create-invalidation --distribution-id ESWVTZYFL873V --paths "/*" --no-cli-pager
 
       # Release headroom and shrink before final lightweight steps & post-job
       - name: Release reserve & shrink

.github/workflows/validate-docs-site.yaml

@@ -130,7 +130,7 @@ jobs:
       run: aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID_STAGING }} && aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY_STAGING }} && aws configure set default.region us-east-1
 
     - name: Deploy PR preview
-      run: aws s3 sync site/_site s3://validmind-docs-staging/site/pr_previews/${{ github.head_ref }} --delete && aws cloudfront create-invalidation --distribution-id ESWVTZYFL873V --paths "/*" --no-cli-pager
+      run: aws s3 sync site/_site s3://validmind-docs-staging/site/pr_previews/${{ github.head_ref }} --delete --exclude "notebooks/EXECUTED/*" && aws cloudfront create-invalidation --distribution-id ESWVTZYFL873V --paths "/*" --no-cli-pager
 
     - name: Post comment with preview URL
       uses: actions/github-script@v6

.gitignore

@@ -36,9 +36,6 @@ site/validmind-docs.yaml
 # Python API docs are now generated on the fly
 site/validmind
 
-# Generated template schema documentation
-site/guide/templates/_template-schema-generated.qmd
-
 # Cursor rules
 .cursor/rules/
 .cursor/skills/

README.md

@@ -150,7 +150,7 @@ Some documentation content is auto-generated from backend source files. These sc
 
 #### Template schema documentation
 
-The template schema reference in `site/guide/templates/customize-document-templates.qmd` is auto-generated from the backend JSON Schema. CI workflows generate this automatically, but you can also regenerate locally:
+The template schema reference in `site/guide/templates/customize-document-templates.qmd` is auto-generated from the backend JSON Schema, overwriting any baseline output checked into this repo. You can also regenerate locally and commit:
 
 ```bash
 cd site
@@ -172,7 +172,7 @@ make template-schema-docs
 The script reads from:
 - `backend/src/backend/templates/documentation/model_documentation/mdd_template_schema_v5.json` — template schema definition
 
-Output: `site/guide/templates/_template-schema-generated.qmd`
+Output: Content is injected directly into `site/guide/templates/customize-document-templates.qmd` between marker comments.
 
 #### Stylesheet organization (IN PROGRESS)
 

scripts/generate_template_schema_docs.py

@@ -30,7 +30,10 @@
 BACKEND_ROOT = Path(os.environ.get("BACKEND_ROOT", REPO_ROOT.parent / "backend"))
 
 SCHEMA_FILE = BACKEND_ROOT / "src/backend/templates/documentation/model_documentation/mdd_template_schema_v5_ui.json"
-OUTPUT_FILE = REPO_ROOT / "site/guide/templates/_template-schema-generated.qmd"
+TARGET_FILE = REPO_ROOT / "site/guide/templates/_template-schema-generated.qmd"
+
+# Minimum expected file size in bytes (sanity check for valid output)
+MIN_OUTPUT_SIZE = 40000
 
 # CSS stylesheets to link in the generated HTML
 STYLESHEETS = [
@@ -62,7 +65,7 @@ def main():
     print(f"Generating schema documentation from {SCHEMA_FILE}")
 
     # Create temporary output
-    temp_output = OUTPUT_FILE.with_suffix(".tmp.html")
+    temp_output = TARGET_FILE.with_suffix(".tmp.html")
 
     subprocess.run([
         "generate-schema-doc",
@@ -98,6 +101,20 @@ def main():
         '',
         html_content
     )
+    # Strip the title tag (not needed when embedded in Quarto page)
+    html_content = re.sub(
+        r'\s*<title>[^<]*</title>\s*',
+        '\n',
+        html_content
+    )
+    # Strip h1 headings (the page already has a heading from Quarto)
+    html_content = re.sub(
+        r'<h1>[^<]*</h1>',
+        '',
+        html_content
+    )
+    # Clean up multiple consecutive blank lines
+    html_content = re.sub(r'\n{3,}', '\n\n', html_content)
 
     # Build stylesheet link tags
     stylesheet_links = '\n'.join(
@@ -124,34 +141,45 @@ def main():
         '</div></body>'
     )
 
-    # Copyright header to place before the raw HTML block
-    copyright_header = """<!-- Copyright © 2023-2026 ValidMind Inc. All rights reserved.
+    # Strip leading/trailing whitespace from HTML content
+    html_content = html_content.strip()
+
+    # Build the output content (raw HTML block for Quarto include)
+    output_content = f"""<!-- Copyright © 2023-2026 ValidMind Inc. All rights reserved.
 Refer to the LICENSE file in the root of this repository for details.
 SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 This file is auto-generated by scripts/generate_template_schema_docs.py
 Do not edit directly. Re-run the script to update.
 
-Source: backend/src/backend/templates/documentation/model_documentation/mdd_template_schema_v5.json
--->"""
-
-    # Wrap HTML in Quarto raw HTML block for .qmd file, with comment before
-    qmd_content = f"""{copyright_header}
+Source: {SCHEMA_FILE.relative_to(BACKEND_ROOT.parent)}
+-->
 
 ```{{=html}}
 {html_content}
 ```
 """
 
-    # Write final output
-    OUTPUT_FILE.parent.mkdir(parents=True, exist_ok=True)
-    with open(OUTPUT_FILE, "w") as f:
-        f.write(qmd_content)
+    # Validate output before writing
+    if len(output_content) < MIN_OUTPUT_SIZE:
+        print(f"Error: Generated output is too small ({len(output_content)} bytes)")
+        print("This likely indicates a generation failure.")
+        temp_output.unlink(missing_ok=True)
+        sys.exit(1)
+
+    if "<html" not in html_content or "</html>" not in html_content:
+        print("Error: Generated output does not contain valid HTML structure")
+        temp_output.unlink(missing_ok=True)
+        sys.exit(1)
+
+    # Write to target file
+    TARGET_FILE.write_text(output_content)
 
     # Clean up temp file
     temp_output.unlink()
 
-    print(f"Generated {OUTPUT_FILE}")
+    print(f"Generated template schema documentation: {TARGET_FILE}")
+    print(f"Output size: {len(output_content)} bytes")
 
 
 if __name__ == "__main__":

site/Makefile

@@ -427,7 +427,7 @@ release-notes:
 template-schema-docs:
 	@echo "\nGenerating template schema documentation ..."
 	@if [ ! -d "$(SRC_ROOT)/backend" ]; then echo "Error: Backend not cloned. Run 'make clone' first."; exit 1; fi
-	@pip install -q json-schema-for-humans
+	@python -m pip install -q json-schema-for-humans
 	@BACKEND_ROOT=$(SRC_ROOT)/backend python ../scripts/generate_template_schema_docs.py
 
 test-descriptions: