From 92c71e8e82957b7ef9f12cd6e92da1f31768b582 Mon Sep 17 00:00:00 2001 From: Nik Richers Date: Tue, 31 Mar 2026 18:50:19 -0700 Subject: [PATCH 1/8] Allow check-in of baseline template schema docs and make CI workflow fail harder if necessary --- .gitignore | 3 - README.md | 4 +- scripts/generate_template_schema_docs.py | 56 +- site/Makefile | 2 +- .../templates/_template-schema-generated.qmd | 1143 +++++++++++++++++ .../customize-document-templates.qmd | 7 - 6 files changed, 1188 insertions(+), 27 deletions(-) create mode 100644 site/guide/templates/_template-schema-generated.qmd diff --git a/.gitignore b/.gitignore index 9684da543c..f4de6a8af5 100644 --- a/.gitignore +++ b/.gitignore @@ -36,9 +36,6 @@ site/validmind-docs.yaml # Python API docs are now generated on the fly site/validmind -# Generated template schema documentation -site/guide/templates/_template-schema-generated.qmd - # Cursor rules .cursor/rules/ .cursor/skills/ diff --git a/README.md b/README.md index 1734a7b31e..3a60c2b1ad 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ Some documentation content is auto-generated from backend source files. These sc #### Template schema documentation -The template schema reference in `site/guide/templates/customize-document-templates.qmd` is auto-generated from the backend JSON Schema. CI workflows generate this automatically, but you can also regenerate locally: +The template schema reference in `site/guide/templates/customize-document-templates.qmd` is auto-generated from the backend JSON Schema. CI workflow generation is pending PAT access to the backend repo — for now, regenerate locally and commit: ```bash cd site @@ -172,7 +172,7 @@ make template-schema-docs The script reads from: - `backend/src/backend/templates/documentation/model_documentation/mdd_template_schema_v5.json` — template schema definition -Output: `site/guide/templates/_template-schema-generated.qmd` +Output: Content is injected directly into `site/guide/templates/customize-document-templates.qmd` between marker comments. #### Stylesheet organization (IN PROGRESS) diff --git a/scripts/generate_template_schema_docs.py b/scripts/generate_template_schema_docs.py index 3becbac57d..0211667ead 100644 --- a/scripts/generate_template_schema_docs.py +++ b/scripts/generate_template_schema_docs.py @@ -30,7 +30,10 @@ BACKEND_ROOT = Path(os.environ.get("BACKEND_ROOT", REPO_ROOT.parent / "backend")) SCHEMA_FILE = BACKEND_ROOT / "src/backend/templates/documentation/model_documentation/mdd_template_schema_v5_ui.json" -OUTPUT_FILE = REPO_ROOT / "site/guide/templates/_template-schema-generated.qmd" +TARGET_FILE = REPO_ROOT / "site/guide/templates/_template-schema-generated.qmd" + +# Minimum expected file size in bytes (sanity check for valid output) +MIN_OUTPUT_SIZE = 1000 # CSS stylesheets to link in the generated HTML STYLESHEETS = [ @@ -62,7 +65,7 @@ def main(): print(f"Generating schema documentation from {SCHEMA_FILE}") # Create temporary output - temp_output = OUTPUT_FILE.with_suffix(".tmp.html") + temp_output = TARGET_FILE.with_suffix(".tmp.html") subprocess.run([ "generate-schema-doc", @@ -98,6 +101,20 @@ def main(): '', html_content ) + # Strip the title tag (not needed when embedded in Quarto page) + html_content = re.sub( + r'\s*[^<]*\s*', + '\n', + html_content + ) + # Strip h1 headings (the page already has a heading from Quarto) + html_content = re.sub( + r'

[^<]*

', + '', + html_content + ) + # Clean up multiple consecutive blank lines + html_content = re.sub(r'\n{3,}', '\n\n', html_content) # Build stylesheet link tags stylesheet_links = '\n'.join( @@ -124,34 +141,45 @@ def main(): '' ) - # Copyright header to place before the raw HTML block - copyright_header = """""" - - # Wrap HTML in Quarto raw HTML block for .qmd file, with comment before - qmd_content = f"""{copyright_header} +Source: {SCHEMA_FILE.relative_to(BACKEND_ROOT.parent)} +--> ```{{=html}} {html_content} ``` """ - # Write final output - OUTPUT_FILE.parent.mkdir(parents=True, exist_ok=True) - with open(OUTPUT_FILE, "w") as f: - f.write(qmd_content) + # Validate output before writing + if len(output_content) < MIN_OUTPUT_SIZE: + print(f"Error: Generated output is too small ({len(output_content)} bytes)") + print("This likely indicates a generation failure.") + temp_output.unlink(missing_ok=True) + sys.exit(1) + + if "" not in html_content: + print("Error: Generated output does not contain valid HTML structure") + temp_output.unlink(missing_ok=True) + sys.exit(1) + + # Write to target file + TARGET_FILE.write_text(output_content) # Clean up temp file temp_output.unlink() - print(f"Generated {OUTPUT_FILE}") + print(f"Generated template schema documentation: {TARGET_FILE}") + print(f"Output size: {len(output_content)} bytes") if __name__ == "__main__": diff --git a/site/Makefile b/site/Makefile index 6773da52d5..d3372f286c 100644 --- a/site/Makefile +++ b/site/Makefile @@ -427,7 +427,7 @@ release-notes: template-schema-docs: @echo "\nGenerating template schema documentation ..." @if [ ! -d "$(SRC_ROOT)/backend" ]; then echo "Error: Backend not cloned. Run 'make clone' first."; exit 1; fi - @pip install -q json-schema-for-humans + @python -m pip install -q json-schema-for-humans @BACKEND_ROOT=$(SRC_ROOT)/backend python ../scripts/generate_template_schema_docs.py test-descriptions: diff --git a/site/guide/templates/_template-schema-generated.qmd b/site/guide/templates/_template-schema-generated.qmd new file mode 100644 index 0000000000..8edc74c349 --- /dev/null +++ b/site/guide/templates/_template-schema-generated.qmd @@ -0,0 +1,1143 @@ + + +```{=html} + + + + + + + + + + + + + + +
+ + +
+ +
Type: array
+

Top-level documentation sections shown in the template editor.

+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + +
root + + + + section
Type: object
+

A section in the document tree.

+ + + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ +
root + + + + root items + + + + id
Type: string
+

Unique identifier for the section.

+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + root items + + + + title
Type: string
+

Title of the section.

+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + root items + + + + description
Type: string
+

Description of the section.

+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + root items + + + + order
Type: integer
+

Order of the section in the navigation menu. By default sections are ordered alphabetically. If order is specified, sections will be ordered by the order value, and then alphabetically.

+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + root items + + + + default_text
Type: string
+

Default text for the section. If set, a metadata content row will be created with this text when installing the template on a given project.

+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + root items + + + + index_only
Type: boolean
+

If true, the section will be displayed in the navigation menu, but it will not be accessible via direct link.

+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + root items + + + + condensed
Type: boolean
+

If true, the section will condense all of its subsections into a single section.

+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + root items + + + + guidelines
Type: array of string
+

Documentation or validation guidelines for the section.

+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + +
root + + + + root items + + + + guidelines + + + + guidelines items
Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + root items + + + + contents
Type: array
+

Contents to be displayed on the section.

+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + +
root + + + + root items + + + + contents + + + + section_contents
Type: object
+

Single content block rendered within a section.

+ + + + + + + + + +
+
+
+

+ +

+
+ +
+
+ +
root + + + + root items + + + + contents + + + + contents items + + + + content_type
Type: enum (of string) Default: "text"
+
+

Must be one of:

+
  • "text"
  • "metadata_text"
  • "dynamic"
  • "metric"
  • "test"
  • "guideline"
  • "assessment_summary"
  • "unit_metric"
+
+ + + + + +
+
Examples:
+
"text"
+
+
"test"
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + root items + + + + contents + + + + contents items + + + + content_id

+

ID of the content to be displayed for the given content type.

+

+ +

+
+ + +
root + + + + root items + + + + contents + + + + contents items + + + + content_id + + + + oneOf + + + + item 0
Type: string
+ + + + + + +
+
Example:
+
"sample_text"
+
+
+
+ + +
root + + + + root items + + + + contents + + + + contents items + + + + content_id + + + + oneOf + + + + item 1
Type: array of string
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + +
root + + + + root items + + + + contents + + + + contents items + + + + content_id + + + + oneOf + + + + item 1 + + + + item 1 items
Type: string
+ + + + + + + +
+

+
Example:
+
[
+    "sample_text",
+    "section_intro"
+]
+
+
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + root items + + + + contents + + + + contents items + + + + options
Type: object
+

Options for the content block.

+ + + + + + +
+
Examples:
+
{
+    "default_text": "This is a sample text block."
+}
+
+
{
+    "metric_id": "metric_1",
+    "title": "Custom Title for Metric 1"
+}
+
+
{
+    "test_id": "adf_test"
+}
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + root items + + + + contents + + + + contents items + + + + options + + + + default_text
Type: string
+

Default text for the content block. Only applicable for text content blocks.

+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + root items + + + + contents + + + + contents items + + + + options + + + + title
Type: string
+

Title of the content block. Only applicable for metric and test content blocks.

+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+

Additional Properties of any type are allowed.

+ +
root + + + + root items + + + + contents + + + + contents items + + + + additionalProperties
Type: object
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + root items + + + + sections
Type: array
+

Nested child sections.

+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + +
root + + + + root items + + + + sections + + + + section
Type: object
+

A section in the document tree.

+Same definition as items +
+
+
+
+
+
+
+
+ +
+ +``` diff --git a/site/guide/templates/customize-document-templates.qmd b/site/guide/templates/customize-document-templates.qmd index 0d17104c7a..18622cbdd0 100644 --- a/site/guide/templates/customize-document-templates.qmd +++ b/site/guide/templates/customize-document-templates.qmd @@ -75,12 +75,6 @@ Once saved, your new template version becomes available for use. ### Template schema - - ::: {.column-page-inset-right} {{< include _template-schema-generated.qmd >}} @@ -173,4 +167,3 @@ c. Finish editing your template, then save a new version. [^9]: [Edit YAML templates](#edit-yaml-templates) [^10]: [Template schema](#template-schema) - From 4e1f6034f82aab2f527e2509c2fb8077fed5b88e Mon Sep 17 00:00:00 2001 From: Nik Richers Date: Tue, 31 Mar 2026 20:12:48 -0700 Subject: [PATCH 2/8] Fix README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3a60c2b1ad..2c188de20c 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ Some documentation content is auto-generated from backend source files. These sc #### Template schema documentation -The template schema reference in `site/guide/templates/customize-document-templates.qmd` is auto-generated from the backend JSON Schema. CI workflow generation is pending PAT access to the backend repo — for now, regenerate locally and commit: +The template schema reference in `site/guide/templates/customize-document-templates.qmd` is auto-generated from the backend JSON Schema, overwriting any baseline output checked into this repo. You can also regenerate locally and commit: ```bash cd site From c7a6d11a16a22e50d29504f4f09a398455d00832 Mon Sep 17 00:00:00 2001 From: Nik Richers Date: Thu, 2 Apr 2026 11:59:43 -0700 Subject: [PATCH 3/8] Exclude notebooks/EXECUTED/* from AWS S3 sync --- .github/workflows/validate-docs-site.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-docs-site.yaml b/.github/workflows/validate-docs-site.yaml index da1d3bc9d1..309a7b302f 100644 --- a/.github/workflows/validate-docs-site.yaml +++ b/.github/workflows/validate-docs-site.yaml @@ -130,7 +130,7 @@ jobs: run: aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID_STAGING }} && aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY_STAGING }} && aws configure set default.region us-east-1 - name: Deploy PR preview - run: aws s3 sync site/_site s3://validmind-docs-staging/site/pr_previews/${{ github.head_ref }} --delete && aws cloudfront create-invalidation --distribution-id ESWVTZYFL873V --paths "/*" --no-cli-pager + run: aws s3 sync site/_site s3://validmind-docs-staging/site/pr_previews/${{ github.head_ref }} --delete --exclude "notebooks/EXECUTED/*" && aws cloudfront create-invalidation --distribution-id ESWVTZYFL873V --paths "/*" --no-cli-pager - name: Post comment with preview URL uses: actions/github-script@v6 From 9eac216ddae94e0b7cd9e16112cc950f4dd246ae Mon Sep 17 00:00:00 2001 From: Nik Richers Date: Thu, 2 Apr 2026 12:23:49 -0700 Subject: [PATCH 4/8] Another test notebook --- site/notebooks/another-test.ipynb | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 site/notebooks/another-test.ipynb diff --git a/site/notebooks/another-test.ipynb b/site/notebooks/another-test.ipynb new file mode 100644 index 0000000000..e8a01018e2 --- /dev/null +++ b/site/notebooks/another-test.ipynb @@ -0,0 +1,27 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "b3c0d025", + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "outputs": [], + "source": [ + "# test notebook\n", + "\n", + "remove after testing" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file From a6c67463fe5ef47ebcab8317f4706a8d54c15414 Mon Sep 17 00:00:00 2001 From: Nik Richers Date: Thu, 2 Apr 2026 12:24:38 -0700 Subject: [PATCH 5/8] Move notebook into EXECUTED path --- site/notebooks/{ => EXECUTED}/another-test.ipynb | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename site/notebooks/{ => EXECUTED}/another-test.ipynb (100%) diff --git a/site/notebooks/another-test.ipynb b/site/notebooks/EXECUTED/another-test.ipynb similarity index 100% rename from site/notebooks/another-test.ipynb rename to site/notebooks/EXECUTED/another-test.ipynb From c793f287a39df2c25999c59c18fd423e75fb0f03 Mon Sep 17 00:00:00 2001 From: Nik Richers Date: Thu, 2 Apr 2026 16:34:55 -0700 Subject: [PATCH 6/8] Also --exclude executed notebooks on staging & prod --- .github/workflows/deploy-docs-prod.yaml | 2 +- .github/workflows/deploy-docs-staging.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-docs-prod.yaml b/.github/workflows/deploy-docs-prod.yaml index c2e62fb39b..24697c2f30 100644 --- a/.github/workflows/deploy-docs-prod.yaml +++ b/.github/workflows/deploy-docs-prod.yaml @@ -117,7 +117,7 @@ jobs: run: aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID_PROD }} && aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY_PROD }} && aws configure set default.region us-east-1 - name: Deploy docs prod site - run: aws s3 sync site/_site s3://validmind-docs-prod/site --delete --exclude "installation/omnibus/*" --exclude "installation/helm-repo/*" --exclude "llm/*" && aws cloudfront create-invalidation --distribution-id E2BGG3USKQTR9W --paths "/*" --no-cli-pager + run: aws s3 sync site/_site s3://validmind-docs-prod/site --delete --exclude "installation/omnibus/*" --exclude "installation/helm-repo/*" --exclude "notebooks/EXECUTED/*" --exclude "llm/*" && aws cloudfront create-invalidation --distribution-id E2BGG3USKQTR9W --paths "/*" --no-cli-pager # Release headroom and shrink before final lightweight steps & post-job - name: Release reserve & shrink diff --git a/.github/workflows/deploy-docs-staging.yaml b/.github/workflows/deploy-docs-staging.yaml index 48e76d348f..4c8f8fb69b 100644 --- a/.github/workflows/deploy-docs-staging.yaml +++ b/.github/workflows/deploy-docs-staging.yaml @@ -120,7 +120,7 @@ jobs: run: aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID_STAGING }} && aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY_STAGING }} && aws configure set default.region us-west-2 - name: Deploy docs staging site - run: aws s3 sync site/_site s3://validmind-docs-staging/site --delete --exclude "installation/helm-repo/*" --exclude "pr_previews/*" --exclude "llm/*" && aws cloudfront create-invalidation --distribution-id ESWVTZYFL873V --paths "/*" --no-cli-pager + run: aws s3 sync site/_site s3://validmind-docs-staging/site --delete --exclude "installation/helm-repo/*" --exclude "pr_previews/*" --exclude "notebooks/EXECUTED/*" --exclude "llm/*" && aws cloudfront create-invalidation --distribution-id ESWVTZYFL873V --paths "/*" --no-cli-pager # Release headroom and shrink before final lightweight steps & post-job - name: Release reserve & shrink From ea56c2b6791268c92e31bf4d6f1410ff317f60a1 Mon Sep 17 00:00:00 2001 From: Nik Richers Date: Thu, 2 Apr 2026 16:36:07 -0700 Subject: [PATCH 7/8] Delete another test notebook --- site/notebooks/EXECUTED/another-test.ipynb | 27 ---------------------- 1 file changed, 27 deletions(-) delete mode 100644 site/notebooks/EXECUTED/another-test.ipynb diff --git a/site/notebooks/EXECUTED/another-test.ipynb b/site/notebooks/EXECUTED/another-test.ipynb deleted file mode 100644 index e8a01018e2..0000000000 --- a/site/notebooks/EXECUTED/another-test.ipynb +++ /dev/null @@ -1,27 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "b3c0d025", - "metadata": { - "vscode": { - "languageId": "plaintext" - } - }, - "outputs": [], - "source": [ - "# test notebook\n", - "\n", - "remove after testing" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} \ No newline at end of file From d070a40cb19798fb1baa63ec4c8723d8dd94b699 Mon Sep 17 00:00:00 2001 From: Nik Richers Date: Thu, 2 Apr 2026 16:37:40 -0700 Subject: [PATCH 8/8] Update scripts/generate_template_schema_docs.py --- scripts/generate_template_schema_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generate_template_schema_docs.py b/scripts/generate_template_schema_docs.py index 0211667ead..c790489403 100644 --- a/scripts/generate_template_schema_docs.py +++ b/scripts/generate_template_schema_docs.py @@ -33,7 +33,7 @@ TARGET_FILE = REPO_ROOT / "site/guide/templates/_template-schema-generated.qmd" # Minimum expected file size in bytes (sanity check for valid output) -MIN_OUTPUT_SIZE = 1000 +MIN_OUTPUT_SIZE = 40000 # CSS stylesheets to link in the generated HTML STYLESHEETS = [