Migrate examples to Jupyter notebooks

.githooks/pre-commit

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Portable null-delimited read (macOS system bash 3.2 lacks `mapfile`).
+notebooks=()
+while IFS= read -r -d '' notebook; do
+  notebooks+=("$notebook")
+done < <(git diff --cached --name-only -z --diff-filter=ACMR -- '*.ipynb')
+if [[ ${#notebooks[@]} -eq 0 ]]; then
+  exit 0
+fi
+
+# Re-staging below adds the whole working-tree file, so refuse to run when a
+# notebook has unstaged changes — otherwise those hunks would be committed too.
+for notebook in "${notebooks[@]}"; do
+  if ! git diff --quiet -- "$notebook"; then
+    echo "error: $notebook has unstaged changes; stage or stash them before committing (notebook hooks can't partially stage)." >&2
+    exit 1
+  fi
+done
+
+python3 scripts/sync_notebook_badges.py "${notebooks[@]}"
+python3 scripts/strip_notebook_outputs.py "${notebooks[@]}"
+
+# Re-stage files in case the stripper changed them.
+git add -- "${notebooks[@]}"

.githooks/pre-push

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Portable null-delimited read (macOS system bash 3.2 lacks `mapfile`).
+notebooks=()
+while IFS= read -r -d '' notebook; do
+  notebooks+=("$notebook")
+done < <(git ls-files -z '*.ipynb')
+if [[ ${#notebooks[@]} -eq 0 ]]; then
+  exit 0
+fi
+
+python3 scripts/sync_notebook_badges.py "${notebooks[@]}"
+python3 scripts/strip_notebook_outputs.py "${notebooks[@]}"
+
+if ! git diff --quiet -- "${notebooks[@]}"; then
+  git add -- "${notebooks[@]}"
+  echo "Notebook badges/outputs were normalized during pre-push. Commit the updated notebooks and push again." >&2
+  exit 1
+fi

.github/workflows/CI.yml

@@ -12,8 +12,26 @@ concurrency:
   cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
 
 jobs:
+  notebook-clean:
+    name: Notebook Output Check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - name: Verify notebooks are stripped
+        run: |
+          notebooks=()
+          while IFS= read -r -d '' notebook; do
+            notebooks+=("$notebook")
+          done < <(git ls-files -z '*.ipynb')
+          if [ ${#notebooks[@]} -eq 0 ]; then
+            exit 0
+          fi
+          python3 scripts/sync_notebook_badges.py "${notebooks[@]}"
+          python3 scripts/strip_notebook_outputs.py "${notebooks[@]}"
+          git diff --exit-code
+
   test:
-    name: ${{ matrix.pkg.name }} - Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }}
+    name: ${{ matrix.pkg.name }} - Julia ${{ matrix.version }} - ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -25,8 +43,6 @@ jobs:
           - ubuntu-latest
           - windows-latest
           - macOS-latest
-        arch:
-          - x64
         pkg:
           - name: SSMProblems
             dir: './SSMProblems'
@@ -40,7 +56,6 @@ jobs:
         uses: julia-actions/setup-julia@v3
         with:
           version: ${{ matrix.version }}
-          arch: ${{ matrix.arch }}
 
       - name: Cache artifacts
         uses: julia-actions/cache@v3
@@ -108,4 +123,4 @@ jobs:
             using CUDA;
             using GeneralisedFilters;
             println("GeneralisedFilters with CUDA loaded successfully");
-            println("CUDA functional: ", CUDA.functional())'
+            println("CUDA functional: ", CUDA.functional())'

.github/workflows/Documentation.yml

@@ -37,6 +37,23 @@ jobs:
         with:
           version: '1'
       - uses: julia-actions/cache@v3
+      - name: Cache notebook outputs
+        if: matrix.pkg.name == 'GeneralisedFilters'
+        uses: actions/cache@v4
+        with:
+          path: GeneralisedFilters/docs/src/examples
+          key: notebooks-${{ runner.os }}-${{ hashFiles('GeneralisedFilters/src/**', 'SSMProblems/src/**', 'GeneralisedFilters/examples/**', 'GeneralisedFilters/docs/*.jl') }}
+      - name: Cache pip packages
+        if: matrix.pkg.name == 'GeneralisedFilters'
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-jupyter-nbconvert
+      - name: Install notebook tooling
+        if: matrix.pkg.name == 'GeneralisedFilters'
+        run: |
+          python3 -m pip install --upgrade pip
+          python3 -m pip install jupyter nbconvert
       - name: Install dependencies
         run: |
           julia --project=${{ matrix.pkg.dir }}/docs/ --color=yes -e '

GeneralisedFilters/docs/Project.toml

@@ -1,3 +1,3 @@
 [deps]
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
-Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306"
+IJulia = "7073ff75-c697-5162-941a-fcdaad2a7d2a"

GeneralisedFilters/docs/make.jl

@@ -4,14 +4,19 @@ push!(LOAD_PATH, "../src/")
 # With minor changes from https://github.com/JuliaGaussianProcesses/AbstractGPs.jl/docs
 #
 ### Process examples
-# Always rerun examples
+const EXAMPLES_ROOT = joinpath(@__DIR__, "..", "examples")
 const EXAMPLES_OUT = joinpath(@__DIR__, "src", "examples")
-ispath(EXAMPLES_OUT) && rm(EXAMPLES_OUT; recursive=true)
 mkpath(EXAMPLES_OUT)
 
 # Install and precompile all packages
 # Workaround for https://github.com/JuliaLang/Pkg.jl/issues/2219
-examples = filter!(isdir, readdir(joinpath(@__DIR__, "..", "examples"); join=true))
+# Only build directories that ship the expected `<slug>.ipynb`; other example dirs
+# (e.g. script-only scratch examples) are skipped rather than failing the build.
+examples = sort(
+    filter(readdir(EXAMPLES_ROOT; join=true)) do path
+        return isdir(path) && isfile(joinpath(path, string(basename(path), ".ipynb")))
+    end,
+)
 above = joinpath(@__DIR__, "..")
 ssmproblems_path = joinpath(above, "..", "SSMProblems")
 let script = "using Pkg; Pkg.activate(ARGS[1]); Pkg.develop(path=\"$(above)\"); Pkg.develop(path=\"$(ssmproblems_path)\"); Pkg.instantiate()"
@@ -26,11 +31,13 @@ let script = "using Pkg; Pkg.activate(ARGS[1]); Pkg.develop(path=\"$(above)\");
     end
 end
 # Run examples asynchronously
-processes = let literatejl = joinpath(@__DIR__, "literate.jl")
+processes = let
+    notebookjl = joinpath(@__DIR__, "notebook.jl")
+    docs_project = abspath(@__DIR__)
     map(examples) do example
         return run(
             pipeline(
-                `$(Base.julia_cmd()) $literatejl $(basename(example)) $EXAMPLES_OUT`;
+                `$(Base.julia_cmd()) --project=$(docs_project) $notebookjl $(basename(example)) $EXAMPLES_OUT`;
                 stdin=devnull,
                 stdout=devnull,
                 stderr=stderr,
@@ -43,6 +50,13 @@ end
 # Check that all examples were run successfully
 isempty(processes) || success(processes) || error("some examples were not run successfully")
 
+const EXAMPLE_MARKDOWNS = sort(
+    filter(
+        filename -> endswith(filename, ".md") && filename != "index.md",
+        readdir(EXAMPLES_OUT),
+    ),
+)
+
 # Building Documenter
 using Documenter
 using GeneralisedFilters
@@ -56,12 +70,7 @@ makedocs(;
     format=Documenter.HTML(; size_threshold=1000 * 2^11), # 1Mb per page
     pages=[
         "Home" => "index.md",
-        "Examples" => [
-            map(
-                (x) -> joinpath("examples", x),
-                filter!(filename -> endswith(filename, ".md"), readdir(EXAMPLES_OUT)),
-            )...,
-        ],
+        "Examples" => map(x -> joinpath("examples", x), EXAMPLE_MARKDOWNS),
     ],
     #strict=true,
     checkdocs=:exports,

GeneralisedFilters/docs/notebook.jl

@@ -0,0 +1,163 @@
+# Build one example from notebook source into markdown for Documenter.
+if length(ARGS) != 2
+    error("please specify the name of the example and the output directory")
+end
+
+const EXAMPLE = ARGS[1]
+const OUTDIR = ARGS[2]
+const REPO = "TuringLang/SSMProblems.jl"
+const PKG_SUBDIR = "GeneralisedFilters"
+const DOCS_ENV = abspath(@__DIR__)
+
+using Base64: base64decode
+using Pkg: Pkg
+
+const EXAMPLEPATH = joinpath(@__DIR__, "..", "examples", EXAMPLE)
+const NOTEBOOK_FILENAME = string(EXAMPLE, ".ipynb")
+const NOTEBOOK = joinpath(EXAMPLEPATH, NOTEBOOK_FILENAME)
+const MARKDOWN = joinpath(OUTDIR, string(EXAMPLE, ".md"))
+
+isfile(NOTEBOOK) || error("example $(EXAMPLE) must include $(NOTEBOOK_FILENAME)")
+
+if isfile(MARKDOWN)
+    @info "Skipping $(EXAMPLE): cached output found at $(MARKDOWN)"
+    exit(0)
+end
+
+# Keep notebook execution in the example's own environment.
+Pkg.activate(EXAMPLEPATH)
+Pkg.instantiate()
+
+Pkg.activate(DOCS_ENV)
+Pkg.instantiate()
+using IJulia
+
+const KERNEL_NAME = "gf-docs-julia-$(VERSION.major).$(VERSION.minor)"
+IJulia.installkernel(KERNEL_NAME, "--project=$(DOCS_ENV)"; specname=KERNEL_NAME)
+
+function run_nbconvert(
+    examplepath::AbstractString,
+    outdir::AbstractString,
+    name::AbstractString,
+    kernel::AbstractString,
+)
+    jupyter = Sys.which("jupyter")
+    isnothing(jupyter) && error(
+        "jupyter executable not found. Install it (e.g. `pip install jupyter nbconvert`) before building docs.",
+    )
+
+    cmd = `$(jupyter) nbconvert --to markdown --execute --ExecutePreprocessor.timeout=3600 --ExecutePreprocessor.kernel_name=$(kernel) --output=$(name) --output-dir=$(outdir) $(NOTEBOOK_FILENAME)`
+    run(pipeline(Cmd(cmd; dir=examplepath); stdin=devnull, stdout=devnull, stderr=stderr))
+    return nothing
+end
+
+function inject_edit_url(markdown_path::AbstractString, example::AbstractString)
+    content = read(markdown_path, String)
+    meta_block = string(
+        "```@meta\n",
+        "EditURL = \"../../../examples/",
+        example,
+        "/",
+        NOTEBOOK_FILENAME,
+        "\"\n",
+        "```\n\n",
+    )
+
+    # Match an existing `@meta` block only at the very start of the page (same
+    # anchor as the replacement); a `@meta` fence elsewhere in the body must not
+    # suppress injection.
+    if occursin(r"(?s)^```@meta.*?```", content)
+        content = replace(content, r"(?s)^```@meta.*?```\s*" => meta_block; count=1)
+    else
+        content = string(meta_block, content)
+    end
+
+    write(markdown_path, content)
+    return nothing
+end
+
+function inject_docs_badges(markdown_path::AbstractString, example::AbstractString)
+    content = read(markdown_path, String)
+    notebook_name = string(example, ".ipynb")
+
+    colab_url = string(
+        "https://colab.research.google.com/github/",
+        REPO,
+        "/blob/main/",
+        PKG_SUBDIR,
+        "/examples/",
+        example,
+        "/",
+        notebook_name,
+    )
+    source_url = string(
+        "https://github.com/",
+        REPO,
+        "/blob/main/",
+        PKG_SUBDIR,
+        "/examples/",
+        example,
+        "/",
+        notebook_name,
+    )
+    badge_line = string(
+        "[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](",
+        colab_url,
+        ") [![View Source](https://img.shields.io/badge/View%20Source-GitHub-181717?logo=github)](",
+        source_url,
+        ")",
+    )
+
+    # Remove existing notebook badge lines, then inject docs-specific badges.
+    content = replace(content, r"(?m)^\[\!\[Open in Colab\].*\n?" => "";)
+    content = replace(content, r"\n{3,}" => "\n\n";)
+
+    heading_match = match(r"(?m)^# .+$", content)
+    if isnothing(heading_match)
+        content = string(badge_line, "\n\n", content)
+    else
+        # Use `nextind`/`prevind` so multibyte headings (e.g. "# Café") don't land
+        # on a UTF-8 continuation byte and throw `StringIndexError`.
+        head_start = heading_match.offset
+        tail_start = nextind(content, head_start + ncodeunits(heading_match.match) - 1)
+        head = content[1:prevind(content, tail_start)]
+        tail = content[tail_start:end]
+        tail = replace(tail, r"^\n+" => "")
+        content = string(head, "\n\n", badge_line, "\n\n", tail)
+    end
+
+    write(markdown_path, content)
+    return nothing
+end
+
+# CairoMakie/IJulia stores figures as text/html containing <img src="data:image/png;base64,...">
+# which nbconvert passes through verbatim. Documenter doesn't render raw HTML, so we extract
+# the base64 data to real files and replace with standard markdown image syntax.
+function extract_inline_images(
+    markdown_path::AbstractString, outdir::AbstractString, example::AbstractString
+)
+    content = read(markdown_path, String)
+    img_dir = joinpath(outdir, string(example, "_files"))
+    counter = 0
+    content = replace(
+        content,
+        r"<img[^>]*\bsrc=\"data:image/([^;]+);base64,([^\"]+)\"[^>]*>" => function (m)
+            inner = match(r"src=\"data:image/([^;]+);base64,([^\"]+)\"", m)
+            isnothing(inner) && return m
+            fmt = replace(inner.captures[1], "svg+xml" => "svg")
+            data = replace(inner.captures[2], r"\s" => "")
+            counter += 1
+            mkpath(img_dir)
+            filename = "$(example)_$(counter).$(fmt)"
+            write(joinpath(img_dir, filename), base64decode(data))
+            return "![]($(example)_files/$(filename))"
+        end,
+    )
+    write(markdown_path, content)
+    return nothing
+end
+
+run_nbconvert(EXAMPLEPATH, OUTDIR, EXAMPLE, KERNEL_NAME)
+extract_inline_images(MARKDOWN, OUTDIR, EXAMPLE)
+inject_docs_badges(MARKDOWN, EXAMPLE)
+inject_edit_url(MARKDOWN, EXAMPLE)

GeneralisedFilters/examples/trend-inflation/Project.toml

@@ -5,7 +5,6 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 GeneralisedFilters = "3ef92589-7ab8-43f9-b5b9-a3a0c86ecbb7"
-Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306"
 LogExpFunctions = "2ab3a3ac-af41-5b50-aa03-7779005ae688"
 PDMats = "90014a1f-27ba-587c-ab20-58faa44d9150"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"