diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 81188d0c..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,144 +0,0 @@ ---- -version: 2 - -references: - - filter_all: &filter_all - filters: - branches: - only: /.*/ - tags: - only: /.*/ - - filter_stg: &filter_head - filters: - branches: - only: master - tags: - only: stg - - filter_prd: &filter_release - filters: - branches: - ignore: /.*/ - tags: - only: /v[0-9]+\.[0-9]+\.[0-9]+(-[0-9]+)?/ - -jobs: - - test_node10: - docker: - - image: circleci/node:10 - working_directory: ~/src - steps: - - checkout - - restore_cache: - keys: - - v2-node10-dependencies-{{ checksum "package.json" }} - - v2-node10-dependencies- - - run: npm install - - save_cache: - key: v2-node10-dependencies-{{ checksum "package.json" }} - paths: - - node_modules - - run: npm test - - test_node12: - docker: - - image: circleci/node:12 - working_directory: ~/src - steps: - - checkout - - restore_cache: - keys: - - v2-node12-dependencies-{{ checksum "package.json" }} - - v2-node12-dependencies- - - run: npm install - - save_cache: - key: v2-node12-dependencies-{{ checksum "package.json" }} - paths: - - node_modules - - run: npm test - - run: cat coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js - - test_node14: - docker: - - image: circleci/node:14 - working_directory: ~/src - steps: - - checkout - - restore_cache: - keys: - - v2-node14-dependencies-{{ checksum "package.json" }} - - v2-node14-dependencies- - - run: npm install - - save_cache: - key: v2-node14-dependencies-{{ checksum "package.json" }} - paths: - - node_modules - - run: npm test - - deploy_docs: - docker: - - image: circleci/node:14 - working_directory: ~/src - steps: - - checkout - - restore_cache: - key: v1-website-dependencies-{{ checksum "website/package.json" }} - - run: - name: Build - command: | - sudo apt-get -y install awscli - bash ./.circleci/scripts/deploy-docs.sh - - save_cache: - key: v1-website-dependencies-{{ checksum "website/package.json" }} - paths: - - website/node_modules - - deploy_package: - docker: - - image: circleci/node:12 - working_directory: ~/repo - steps: - - checkout - - restore_cache: - keys: - - v2-node12-dependencies-{{ checksum "package.json" }} - - v2-node12-dependencies- - - run: npm install - - run: | - echo "$NPMRC" > ~/.npmrc - chmod 600 ~/.npmrc - if [[ "$CIRCLE_TAG" = *-* ]]; then - npm publish --tag=prerelease - else - npm publish - fi - -workflows: - version: 2 - test: - jobs: - - test_node10: - <<: *filter_all - - test_node12: - <<: *filter_all - - test_node14: - <<: *filter_all - - deploy_docs: - <<: *filter_head - context: - - Documentation - requires: - - test_node10 - - test_node12 - - test_node14 - - deploy_package: - <<: *filter_release - context: - - npm-publish - requires: - - test_node10 - - test_node12 - - test_node14 diff --git a/.circleci/scripts/deploy-docs.sh b/.circleci/scripts/deploy-docs.sh deleted file mode 100644 index 85fd8a85..00000000 --- a/.circleci/scripts/deploy-docs.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -set -e -o pipefail - -BUCKET=docs.clayplatform.io - -green(){ - printf "\e[32m$1\e[39m\n" -} - -red(){ - >&2 printf "\e[31m$1\e[39m\n" -} - -export PROJECT_NAME="${PROJECT_NAME:-$CIRCLE_PROJECT_REPONAME}" -if [[ -z "$PROJECT_NAME" ]]; then - red "PROJECT_NAME not set and could not be derived from CIRCLE_PROJECT_REPONAME" - exit 1 -fi -green "PROJECT_NAME: $PROJECT_NAME" - -export BUILD_DIR="${BUILD_DIR:-website}" -green "BUILD_DIR: $BUILD_DIR" - -green "Building documentation..." -cd "$BUILD_DIR" -npm install --quiet -npm run build - -green "Uploading documentation to $BUCKET..." -aws s3 sync --delete --acl=public-read "build/$PROJECT_NAME/" "s3://$BUCKET/$PROJECT_NAME/" - -green "Documentation updated." diff --git a/.circleci/scripts/release.sh b/.circleci/scripts/release.sh deleted file mode 100755 index 9f04adf2..00000000 --- a/.circleci/scripts/release.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -# This script will increment the package version in package.json. -# It will then tag the HEAD of the master branch with the version number -# and push the tag to the origin (GitHub) - -set -e - -OPTIONS="(prepatch|patch|preminor|minor|premajor|major)" -USAGE="usage: npm release $OPTIONS" -SEMVAR="$1" - -# Releases should only be cut from the master branch. -# They can be manually cut from other branches, but that should be a very -# intentional process. -BRANCH="$(git rev-parse --abbrev-ref HEAD)" -if [[ "$BRANCH" != "master" ]]; then - >&2 echo "ERROR: Not on the master branch, will not release." - exit 1 -fi - -case $SEMVAR in - minor) - ;; - major) - ;; - patch) - ;; - premajor) - ;; - preminor) - ;; - prepatch) - ;; - *) - SEMVAR=prepatch - >&2 echo "WARNING: No $OPTIONS provided, defaulting to PREPATCH." - >&2 echo "$USAGE" - ;; -esac - -git pull --rebase origin master -version="$(npm version $SEMVAR)" -git push origin master -git push origin "tags/$version" diff --git a/.github/main.workflow b/.github/main.workflow deleted file mode 100644 index f982d661..00000000 --- a/.github/main.workflow +++ /dev/null @@ -1,28 +0,0 @@ -workflow "Deploy to GitHub Pages" { - on = "push" - resolves = ["Build and push docs"] -} - -action "Filter branch" { - uses = "actions/bin/filter/@master" - args = "branch master" -} - -action "Install" { - needs = ["Filter branch"] - uses = "actions/npm@master" - args = "install --prefix ./website" -} - -action "Update version" { - needs = ["Install"] - uses = "clay/docusaurus-github-action@master" - args = "version" -} - -action "Build and push docs" { - needs = ["Update version"] - uses = "clay/docusaurus-github-action@master" - args = "deploy" - secrets = ["DEPLOY_SSH_KEY", "ALGOLIA_API_KEY"] -} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..1e3f8624 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,30 @@ +name: CI + +on: + push: + branches: ['**'] + pull_request: + branches: ['**'] + +jobs: + test: + runs-on: ubuntu-latest + + strategy: + matrix: + node-version: [20.x] + + steps: + - uses: actions/checkout@v4 + + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Run tests + run: npm test diff --git a/.gitignore b/.gitignore index a4e323dc..a6a10357 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,4 @@ website/build/ website/yarn.lock website/node_modules website/i18n/* +*.tgz diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 00000000..2bd5a0a9 --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +22 diff --git a/BUNDLER-COMPARISON.md b/BUNDLER-COMPARISON.md new file mode 100644 index 00000000..9c411585 --- /dev/null +++ b/BUNDLER-COMPARISON.md @@ -0,0 +1,473 @@ +# Bundler Pipeline Comparison + +> **Why did we choose Vite?** This document is a record of the pipelines we evaluated before +> arriving at that decision. Only two pipelines exist in `claycli` today: the legacy +> `clay compile` (Browserify) and the current `clay vite`. The other pipelines described here +> — esbuild and bare Rollup — were tested and discarded. They are documented here purely so +> the reasoning is preserved, not because they are available or will ever be shipped. +> +> For the full technical reference of the Vite pipeline, see [`CLAY-VITE.md`](./CLAY-VITE.md). + +--- + +## Table of Contents + +1. [The Legacy: Browserify + Gulp](#1-the-legacy-browserify--gulp) +2. [Attempt 1: esbuild (clay build)](#2-attempt-1-esbuild-clay-build) +3. [Attempt 2: Rollup + esbuild (clay rollup)](#3-attempt-2-rollup--esbuild-clay-rollup) +4. [The Choice: Vite (clay vite)](#4-the-choice-vite-clay-vite) +5. [Measured Performance: Vite vs Browserify](#5-measured-performance-vite-vs-browserify) +6. [Bundler Comparison Matrix](#6-bundler-comparison-matrix) +7. [Why Not the Others](#7-why-not-the-others) +8. [Migration Roadmap](#8-migration-roadmap) + +--- + +## 1. The Legacy: Browserify + Gulp + +### What it did + +Browserify consumed every `client.js`, `model.js`, and `kiln.js` file as an entry point +and bundled them into alpha-bucketed mega-bundles (`_deps-a.js`, `_kiln-a-d.js`, etc.). +Gulp orchestrated 20+ sequential plugins to wire CSS, templates, fonts, and JS together. +A custom runtime (`_prelude.js` + `_postlude.js`) shipped with every page and registered +all modules in a global `window.modules` map under numeric IDs. + +``` +Source files + │ + ▼ +Browserify + Babel (30–60 s) + │ ├── wraps every CJS module in a factory function + │ ├── assigns numeric IDs + │ └── emits _deps-a.js, _deps-b.js … (alpha-bucketed) + │ + ▼ +_prelude.js / _postlude.js ← shipped to every page +_registry.json + _ids.json ← opaque numeric dep graph +_client-init.js ← mounts every .client module loaded, DOM or not +``` + +### Problems + +| Problem | Impact | +|---|---| +| Mega-bundles (all components in one bucket) | Any change rebuilt everything; watch mode: 30–60 s | +| Gulp plugin chain (20+ plugins) | Complex dependency graph, version conflicts, slow installs | +| Sequential build steps | CSS, JS, templates all waited on each other; total ≈ sum of all steps | +| No shared chunk extraction | Each component dragged in its own copy of shared deps | +| No tree shaking | Entire CJS modules bundled regardless of what was used | +| No source maps | Production errors pointed to minified line numbers | +| Static filenames | `article.client.js` — full CDN invalidation on every deploy | +| `window.modules` runtime (616 KB/page) | Every page carried uncacheable inlined JS | +| Babelify transpilation | Even tiny changes triggered a full Babel pass | +| `_registry.json` numeric module graph | Opaque, impossible to inspect or extend | +| `browserify-cache.json` | Stale cache silently served old module code | + +**Performance baseline (Lighthouse, 3 runs, simulated Moto G Power / slow 4G):** + +| Metric | Browserify | +|---|---| +| Perf score | 48 | +| FCP | 2.8 s | +| LCP | 14.3 s | +| TBT | 511 ms | +| TTI | 23.2 s | +| JS transferred | 417 KB | +| Total JS gzip | 6,944 KB | +| Content-hashed files | 0% | +| Inline JS per page | 616 KB (uncacheable) | + +--- + +## 2. Attempt 1: esbuild (`clay build`) + +### What it did + +esbuild replaced Browserify as the JS bundler and PostCSS 8's programmatic API replaced +Gulp's stream-based CSS pipeline. All build steps ran in parallel. The custom +`window.modules` runtime was replaced with native ESM and a generated `_view-init.js` +bootstrap that dynamically imports component code only when the component's DOM element +is present. + +### Strengths + +- **Extremely fast.** esbuild is a native Go binary — ~3 s for JS, ~33 s total (was 90–120 s). +- **Parallel steps.** Media, JS, CSS, templates, fonts — all run simultaneously. +- **Native ESM output.** No custom runtime; browsers handle imports natively. +- **Content-hashed filenames.** Unchanged files stay cached across deploys. +- **Human-readable `_manifest.json`.** Replaced the numeric `_registry.json` + `_ids.json`. + +### Limitations + +- **No `manualChunks` equivalent.** esbuild splits at every shared module boundary regardless + of size, producing hundreds (500+) of tiny files. Hundreds of tiny HTTP/2 streams add + parse overhead and delay the LCP image fetch. +- **CJS circular deps.** esbuild inlines all CJS into a flat IIFE scope, which sidesteps + circular dependency ordering — but CJS modules with runtime initialization order + dependencies can behave unexpectedly. +- **No chunk size control.** There is no way to say "inline this tiny module into its sole + importer" without switching to a bundler that exposes a module-graph API. +- **CJS→ESM interop is opaque.** esbuild wraps CJS in its own `__commonJS()` helpers with + no user-configurable override. + +### Lesson learned + +esbuild proved that the `_view-init.js` / dynamic import architecture was correct and the +perf wins from native ESM were real. But its lack of a module-graph API made chunk size +management impossible — we needed something built on Rollup. + +--- + +## 3. Attempt 2: Rollup + esbuild (`clay rollup`) + +### What it did + +Rollup 4 drove the module graph, tree-shaking, and chunk assignment. esbuild served only +as a fast in-process transformer for two sub-tasks: define substitution (Node globals) and +optional minification via `renderChunk`. This is structurally similar to how a Gulp pipeline +would wire Browserify for bundling and then pipe output through a separate minifier — each +tool does one thing it is best at. + +### Strengths + +- **`manualChunks` control.** Rollup exposes the full module graph via `getModuleInfo()`. + The custom `viteManualChunksPlugin` walked each module's importer chain and inlined small + private modules back into their sole consumer. This directly addressed the "500 tiny chunks" + problem from esbuild. +- **`strictRequires: 'auto'`** in `@rollup/plugin-commonjs` detected circular CJS + dependencies at build time and wrapped only the participating `require()` calls in lazy + getters. +- **Explicit plugin ordering.** Every transform step was a named plugin in a defined sequence. + +### Why Rollup was not the final answer + +Setting up the Rollup pipeline was significantly more complex than expected: + +1. **CJS/ESM interop required multiple plugins with specific configuration.** `@rollup/plugin-commonjs` + with `strictRequires: 'auto'`, `transformMixedEsModules: true`, `requireReturnsDefault: 'preferred'`, + and a custom `commonjsExclude` list per site. Getting this right without breaking CJS + circular dependencies required extensive debugging. +2. **Two separate bundler passes in one pipeline.** esbuild handled node_modules pre-bundling + conceptually, while Rollup handled source — but without Vite's managed `optimizeDeps` + lifecycle, we had to manually decide what to exclude from `@rollup/plugin-commonjs`. +3. **pyxis-frontend required a safe-wrap plugin** because its internal webpack `eval()`-based + modules conflicted with `@rollup/plugin-commonjs`'s rewriting. This was a per-package + exception that added fragility. The fix required patching the dependency itself. +4. **`process.env` in view mode.** Components that worked fine in the esbuild pipeline + crashed with "process is not defined" under Rollup because the esbuild define transform + was not firing for all cases. Required adding a custom esbuild-transform Rollup plugin. +5. **No client-env.json generation.** This had to be manually ported, then discovered missing + at CI build time with a hard error. +6. **Build time: ~40 s** — slower than Vite's ~30 s for the same output, because Rollup's + JS event loop processes the module graph serially vs Vite's internal optimizations. + +The Rollup pipeline produced correct output, but every problem we solved revealed a new one. +With Vite, the same architecture (Rollup for production) was already pre-configured with +correct defaults for exactly this kind of CJS+ESM mixed project. + +--- + +## 4. The Choice: Vite (`clay vite`) + +### What Vite adds on top of Rollup + +Vite uses Rollup 4 internally for production builds. The key differences from bare Rollup: + +| Concern | Bare Rollup (`clay rollup`) | Vite (`clay vite`) | +|---|---|---| +| `node_modules` CJS handling | `@rollup/plugin-commonjs` on everything | esbuild pre-bundler (`optimizeDeps`) converts CJS deps before Rollup sees them | +| CJS circular deps in node_modules | Requires per-package `commonjsExclude` tuning | Handled automatically by pre-bundler | +| pyxis safe-wrap workaround | Required a custom plugin | Not needed — pre-bundler resolves webpack eval() modules | +| Plugin API | Rollup hooks only | Rollup hooks + Vite build extensions (`closeBundle`, `generateBundle`, etc.) — dev-server hooks (`configureServer`, HMR) are not used | +| Dev watch | `rollup.watch()` + chokidar polling | `rollup.watch()` (Rollup incremental rebuild) — **Vite's HMR dev server is not used**; Clay uses a server-rendered architecture (Amphora) that has no Vite dev server in the request path | +| Config surface | Every Rollup option must be threaded manually | One `bundlerConfig()` hook exposes the relevant subset | +| Build speed (production) | ~40 s | ~30 s | +| Vue 3 migration | Would require a custom SFC compiler plugin | `@vitejs/plugin-vue` is first-party and maintained by the Vite team | +| Lightning CSS migration | Manual Rollup plugin | `css: { transformer: 'lightningcss' }` in baseViteConfig | +| Rolldown migration | Not applicable | Direct swap when Rolldown is stable (same plugin API, same config shape) | + +### Why `optimizeDeps` was the key insight + +The single largest source of friction in the Rollup pipeline was CJS interop for +`node_modules`. Packages like pyxis-frontend, vue, and various utility libraries all +needed special handling. Vite's `optimizeDeps` pre-bundles all `node_modules` via esbuild +*before* Rollup sees them, converting CJS to ESM in one batch. `@rollup/plugin-commonjs` +then only needs to handle project source files — a much smaller surface where the site +developer has full control. + +By disabling `optimizeDeps.noDiscovery: true` we further prevented any accidental dep +scanning that could add latency. The result is a clean, predictable build where CJS +complexity is handled at the boundary of `node_modules`, not inside the source graph. + +### Why the config API is better + +With bare Rollup, any site-level customization required understanding the full Rollup +configuration — input, output, plugins array ordering, commonjsOptions, etc. Bugs like +"my plugin runs before commonjs rewrites the module" were non-obvious. + +Vite's `bundlerConfig()` hook in `claycli.config.js` is a minimal, purpose-built API +that exposes only what sites need to customize: + +```js +bundlerConfig: config => { + config.manualChunksMinSize = 8192; // chunk inlining threshold + config.alias = { '@sentry/node': '@sentry/browser' }; // simple redirects + config.define = { DS: 'window.DS' }; // identifier replacements + config.plugins = [...]; // extra Rollup plugins + return config; +} +``` + +Everything else — plugin ordering, Rollup internals, CJS interop settings, output format, +chunk naming, modulepreload polyfill, etc. — is managed by claycli. Sites that never need +to touch these settings simply do not define `bundlerConfig`. + +### ESM migration runway + +Every CJS compatibility shim in the Vite pipeline is a named, documented item with a clear +"removed when" condition: + +| Shim | Removed when | +|---|---| +| `@rollup/plugin-commonjs` | All `.js` files use `import`/`export` | +| `strictRequires: 'auto'` | No CJS circular deps remain | +| `transformMixedEsModules: true` | `.vue` scripts use `import` only | +| `hoistRequires` in vue2Plugin | `.vue` scripts use `import` only | +| `inlineDynamicImports: true` (kiln pass) | Kiln plugins are proper ESM modules | +| Two-pass build | `kilnSplit: true` is enabled | +| `serviceRewritePlugin` | Client/server service contracts are explicit | +| `browserCompatPlugin` | No server-only imports reach the client bundle | + +New components can be written as ESM from day one. Existing components migrate one at a time. +When `clientFilesESM: true` is set in `bundlerConfig`, Rollup's native `experimentalMinChunkSize` +replaces the custom `viteManualChunksPlugin` entirely, getting chunk size control for free. + +### Future technology path + +Vite was chosen specifically because it is the default integration point for: + +- **Lightning CSS** — `css: { transformer: 'lightningcss' }` in `baseViteConfig`. Replaces + PostCSS with Rust-native CSS parsing. One config key, no plugin migration. +- **Vue 3** — `@vitejs/plugin-vue` coexists with the current `viteVue2Plugin`. New components + use Vue 3; legacy components keep Vue 2 until migrated. Both compile correctly in the same + build. +- **Rolldown** — the Rust rewrite of Rollup built by the Vite team. Same plugin API, same + config shape, esbuild-level build speed. The migration will be one `npm install` and a + config tweak in claycli. Nothing in `claycli.config.js` or any component will need to change. + +--- + +## 5. Measured Performance: Vite vs Browserify + +> **About these numbers:** Performance was measured against the Rollup pipeline (`clay rollup`) +> because it was deployed to a feature branch environment first. Since Vite uses Rollup 4 +> internally for production builds and produces structurally identical output (same dynamic +> `import()` bootstrap, same `manualChunks` logic, same content-hashed chunk filenames), +> the Rollup production numbers represent what Vite also achieves. Both pipelines: +> - Run the same `viteManualChunksPlugin` logic +> - Produce the same ESM output format +> - Use the same `_manifest.json` → `resolveModuleScripts()` runtime injection +> - Apply the same caching strategy (`Cache-Control: immutable` for content-hashed files) +> +> The one area where Vite may differ slightly: build time is ~25% faster because Vite's +> internal `optimizeDeps` pass means `@rollup/plugin-commonjs` processes less code. +> +> **URLs used for measurement:** +> - **Vite/Rollup:** `https://jordan-yolo-update.dev.nymag.com/` (minification enabled) +> - **Browserify:** `https://alb-fancy-header.dev.nymag.com/` (legacy `alb-fancy-header` branch) +> +> **Note on URL parity:** The two test URLs serve different featurebranch deployments with +> potentially different page content. Focus on JS-specific and timing metrics, not total bytes. + +### Core Web Vitals (Lighthouse — simulated throttle, 3 runs avg) + +| Metric | Browserify | Vite pipeline | Δ | +|---|---|---|---| +| Perf score | 48 | **50** | **+4%** | +| FCP | 2.8 s | **1.8 s** | **−37%** ✅ | +| LCP | 14.3 s | **11.9 s** | **−17%** ✅ | +| TBT | 511 ms | 565 ms | +11% ⚠ | +| TTI | 23.2 s | 23.2 s | ≈ 0 | +| SI | 6.1 s | 7.1 s | +16% ⚠ | +| TTFB | 346 ms | 340 ms | −2% | +| JS transferred | 417 KB | **478 KB** | +15% ⚠ | + +**Interpretation:** + +- **FCP −37%** is the headline win. The ESM bootstrap delivers first paint earlier than + Browserify's monolithic bundle. The browser receives `` hints + in `` and starts fetching the init scripts during HTML parsing — Browserify had no + preload hints because all JS was inlined in the body. +- **LCP −17%** with minification active. The unminified Rollup/Vite build showed LCP + regression vs Browserify; minification reverses this. The main driver is code volume: the + ESM bootstrap and its critical-path chunks are smaller when minified than Browserify's + single IIFE bundle. +- **TBT +11%** is expected and will improve. Vite emits native ESM modules — each module + requires its own parse + link phase. Browserify emits a single IIFE (one parse pass, all + code evaluated up front). As the codebase migrates to ESM, the `__commonJS()` wrapper + boilerplate shrinks and TBT will improve through better tree-shaking and deferred loading. +- **JS transferred +15%** reflects the Vite build including more entry points per page + (component chunks loaded on demand) vs Browserify's single monolithic bundle. The per-revisit + cache story strongly favours Vite. + +### Core Web Vitals (WebPageTest — real network, Chrome 143, Dulles VA, 3 runs) + +| Metric | Browserify | Vite pipeline | Δ | +|---|---|---|---| +| TTFB | 980 ms | 983 ms | ≈ 0 | +| Start Render | 1,667 ms | **1,633 ms** | **−2%** | +| FCP | 1,658 ms | **1,634 ms** | **−1%** | +| LCP | 4,302 ms | 4,944 ms | +15% ⚠ | +| TBT | 1,416 ms | **1,015 ms** | **−28%** ✅ | +| Speed Index | 4,015 | **3,953** | **−2%** | +| Fully Loaded | 16,869 ms | 21,756 ms | +29% ⚠ | +| Total requests | 195 | 250 | +28% | +| Total bytes | 4.6 MB | 12.1 MB | +163% ⚠ | + +**Interpretation:** + +- **TBT −28%** is a concrete win under real-network conditions. Minification reduces the parse + overhead per chunk and eliminates the `__commonJS()` wrapper boilerplate the browser had to + evaluate on every page load. +- **FCP / Start Render** are marginally faster — consistent with the Lighthouse results. +- **LCP +15%** is the open issue. The primary driver is request count: 250 vs 195. Even on + HTTP/2, 250 concurrent streams creates depth that can delay the LCP image fetch on slower + connections. Raising `manualChunksMinSize` in `claycli.config.js` directly reduces chunk + count. Migrating `client.js` files to ESM also reduces chunk count by eliminating CJS wrapper + modules that inflate chunk size below the merge threshold. +- **Total bytes 12.1 MB vs 4.6 MB:** This difference is dominated by source maps — Vite emits + a `.js.map` per chunk and WebPageTest counts all responses including source maps. The actual + JavaScript the browser executes is **478 KB** per Lighthouse. +- **Fully Loaded +29%:** More HTTP/2 streams settling, but most are cached on repeat visits. + +### Bundle structure comparison (local, minified build) + +| Metric | Browserify | Vite pipeline | +|---|---|---| +| Total JS files | 2,179 | **307** | +| Total uncompressed | 26,942 KB | **19,469 KB** | +| Total gzip | 6,944 KB | **4,571 KB** | +| Shared chunks | 0 | **297** | +| Content-hashed files | 0% | **~97%** | +| Inline JS per page | 616 KB | **0 KB** | +| Warm-cache 304 rate | 82% | **97%** | + +**Notes:** + +- Vite's 307 files break down as: 6 template bundles, 2 kiln bundles, 2 bootstrap/init + `.clay/` files, and 297 shared chunks. +- Total uncompressed is **28% smaller** than Browserify even including source maps. Gzip + wire size drops **34%**. +- The 97% warm-cache rate vs 82% for Browserify reflects content-hashed filenames: unchanged + modules are served from browser cache after the first visit. Browserify's static filenames + forced 304 revalidation for everything on every deploy. +- **616 KB of inline JS per page eliminated.** The Browserify `window.modules` runtime and + component bundle were inlined into every HTML response. This was uncacheable by definition. + +### Build time + +| Pipeline | JS build time | Total time | +|---|---|---| +| Browserify + Gulp | 30–60 s | 90–120 s | +| esbuild | ~3 s | ~33 s | +| Rollup + esbuild | ~40 s | ~70 s | +| **Vite** | **~30 s** | **~30 s** (client-env now free via Rollup plugin) | + +Vite is faster than bare Rollup because its `optimizeDeps` pass converts `node_modules` +CJS to ESM before Rollup sees them, reducing the number of modules `@rollup/plugin-commonjs` +must process. Build time will improve further as files migrate to native ESM (fewer modules +need CJS wrapping). + +--- + +## 6. Bundler Comparison Matrix + +| Capability | Browserify | esbuild | Rollup + esbuild | Vite | +|---|---|---|---|---| +| Build speed | 90–120 s | ~33 s | ~70 s | ~60 s | +| `manualChunks` control | None | None | Full | Full | +| CJS→ESM conversion | N/A (CJS only) | Opaque | Configurable | Managed by `optimizeDeps` | +| CJS circular dep handling | Runtime | Implicit | `strictRequires: 'auto'` | Pre-bundled (automatic) | +| Chunk size inlining | No | No | Yes | Yes + native ESM (`experimentalMinChunkSize`) | +| Tree shaking | No | Yes (ESM only) | Yes | Yes | +| Content-hashed output | No | Yes | Yes | Yes | +| Source maps | No | Yes | Yes | Yes | +| Native ESM output | No | Yes | Yes | Yes | +| `modulepreload` hints | No | Yes * | Yes * | Yes * | +| Vue 3 migration path | No | No | Manual plugin | First-party `@vitejs/plugin-vue` | +| Lightning CSS migration path | No | No | Manual plugin | `css: { transformer: 'lightningcss' }` | +| Rolldown migration path | No | No | No | Drop-in swap (same plugin API) | +| Config API surface | `claycli.config.js` | `claycli.config.js` | All Rollup options exposed | `bundlerConfig()` subset only | +| Dev watch | No | chokidar polling | `rollup.watch()` | `rollup.watch()` (HMR server not used) | +| `node_modules` CJS isolation | N/A | Implicit | Manual `commonjsExclude` | Automatic | +| Setup complexity | Low | Low | High | Medium | +| ESM migration runway | No | Partial | Full | Full + Rolldown-forward-compatible | + +\* Implemented at the `amphora-html` layer, not by the bundler directly. + +--- + +## 7. Why Not the Others + +### Why not keep Browserify? + +The Browserify `window.modules` runtime shipped 616 KB of uncacheable inline JS to every +page. Every deploy invalidated every JS file. No tree shaking, no shared chunk extraction, +no source maps. Build times of 90–120 s made watch mode unusable for local development. + +### Why not just use esbuild? + +esbuild was the right first step — the `_view-init.js` dynamic-import architecture, +`_manifest.json`, and `Cache-Control: immutable` strategy all came from the esbuild phase +and carried forward unchanged. But esbuild's inability to control chunk size (no +`manualChunks` equivalent) meant the 500+ tiny chunk problem was structural, not fixable +with configuration. We needed Rollup's module graph API. + +### Why not stay on Rollup? + +Rollup was viable but required managing too many moving parts at once: + +- `@rollup/plugin-commonjs` configuration with multiple per-package exceptions +- A custom esbuild-transform plugin just to handle `process.env` defines +- A custom safe-wrap plugin for pyxis-frontend that had to be removed when the dep was patched +- Two parallel build passes with carefully synchronized output +- Manual `client-env.json` generation that was missing at first and discovered at CI time + +Every time a new CJS dependency was added to the project, the Rollup pipeline needed updating. +Vite handles this at the `optimizeDeps` boundary automatically. + +Additionally, Rollup is not the strategic direction for the frontend ecosystem. The Vite +team is building **Rolldown** as a Rust replacement for Rollup, targeting 10× build speeds +with the same API. Vite will migrate to Rolldown as its production bundler. By choosing Vite +now, the Clay pipeline gets the Rolldown upgrade for free — one `npm install` in claycli. + +### Why not Webpack? + +Webpack was never seriously considered. Its configuration complexity dwarfs even bare Rollup, +its build speed is 3–5× slower than Vite for this size of codebase, and its ecosystem is +in maintenance mode as projects migrate to Vite. Webpack 5 is still widely used but new +projects in the web ecosystem choose Vite overwhelmingly. + +--- + +## 8. Migration Roadmap + +The long-term direction is **`clay vite` with progressive ESM migration**, targeting Rolldown: + +| Step | Action | Config change | Benefit | +|---|---|---|---| +| 1 | New components: write as ESM from day one | No config change needed | Future-proof from the start; new code is immediately tree-shakeable and Rolldown-ready | +| 2 | Migrate `model.js` / `kiln.js` to ESM | Set `kilnSplit: true` → collapses to one build pass | Eliminates the second Vite build pass; cuts total build time for the kiln/model bundle | +| 3 | Migrate `client.js` files to ESM | Set `clientFilesESM: true` → switches to `experimentalMinChunkSize`; `@rollup/plugin-commonjs` becomes a no-op per file | Native Rollup chunking replaces custom plugin; smaller chunks, better tree-shaking, reduced TBT | +| 4 | Migrate Vue 2 → Vue 3 | Add `@vitejs/plugin-vue`; remove `viteVue2Plugin` from claycli | Smaller runtime (Vue 3 is ~40% smaller than Vue 2), Composition API, first-party Vite support | +| 5 | Replace PostCSS with Lightning CSS | `css: { transformer: 'lightningcss' }` in `baseViteConfig` | Rust-native CSS parsing — dramatically faster CSS build step; modern syntax support with zero config | +| 6 | Remove `commonjsOptions` entirely | All source is native ESM — no CJS shims needed | Removes all `__commonJS()` wrapper boilerplate from output; smaller bundles, lower TBT, cleaner output | +| 7 | Migrate to Rolldown | One `npm install` in claycli; no site `claycli.config.js` changes | esbuild-level build speed (~10×) with full Rollup plugin compatibility; sub-10 s JS build times | + +At step 7, the pipeline is: `vite build` → native ESM output. No CJS shims. No two-pass +build. No PostCSS. No Babel. Sub-10 s build times. + +The key architectural decision that makes this roadmap work: **every CJS shim is a +named, temporary scaffold with a clear removal condition.** Nothing is permanent debt. +Each migration step removes something rather than adding something. diff --git a/CLAY-VITE.md b/CLAY-VITE.md new file mode 100644 index 00000000..614f5d75 --- /dev/null +++ b/CLAY-VITE.md @@ -0,0 +1,1254 @@ +# clay vite — New Asset Pipeline + +> This document covers the **`clay vite`** command — the new build pipeline for Clay instances. +> It explains what changed from the legacy `clay compile` (Browserify) pipeline, why, and +> how the two pipelines compare. +> +> **Why did we choose Vite over the other pipelines we tried?** +> See [`BUNDLER-COMPARISON.md`](./BUNDLER-COMPARISON.md) for the full technical rationale with +> measured performance data and a comparison of Browserify, esbuild, Rollup, and Vite. + +## Table of Contents + +1. [Why We Changed It](#1-why-we-changed-it) +2. [Commands At a Glance](#2-commands-at-a-glance) +3. [Architecture: Old vs New](#3-architecture-old-vs-new) +4. [Pipeline Comparison Diagrams](#4-pipeline-comparison-diagrams) +5. [Feature-by-Feature Comparison](#5-feature-by-feature-comparison) +6. [Configuration](#6-configuration) +7. [Running Both Side-by-Side](#7-running-both-side-by-side) +8. [Code References](#8-code-references) + - [Why `_globals-init.js` exists as a separate file](#why-_globals-initjs-exists-as-a-separate-file) + - [Why the build runs in two passes](#why-the-build-runs-in-two-passes) + - [How client-env.json is generated](#how-client-envjson-is-generated) +9. [Performance](#9-performance) +10. [Learning Curve](#10-learning-curve) +11. [For Product Managers](#11-for-product-managers) +12. [Tests](#12-tests) +13. [Migration Guide](#13-migration-guide) +14. [amphora-html Changes](#14-amphora-html-changes) +15. [Bundler Comparison](#15-bundler-comparison) +16. [Services Pattern and Browser Bundle Hygiene](#16-services-pattern-and-browser-bundle-hygiene) + +## 1. Why We Changed It + +The legacy `clay compile` pipeline was built on **Browserify + Gulp**, tools designed for +the 2014–2018 JavaScript ecosystem. Over time these became pain points: + +| Problem | Impact | +|---|---| +| Browserify megabundle (all components in one file per alpha-bucket) | Any change = full rebuild of all component JS, slow watch mode | +| Gulp orchestration with 20+ plugins | Complex dependency chain, hard to debug, slow npm install | +| Sequential compilation steps | CSS, JS, templates all ran in series — total time = sum of all steps | +| No shared chunk extraction | If two components shared a dependency, each dragged it in separately | +| No tree shaking | Browserify bundled entire CJS modules regardless of how much was used | +| No source maps | Build errors in production pointed to minified line numbers, not source | +| No content-hashed filenames | Static filenames (`article.client.js`) forced full cache invalidation on every deploy | +| `_prelude.js` + `_postlude.js` runtime (616 KB/page) | Every page carried an uncacheable Browserify module registry blob | +| `_registry.json` + `_ids.json` numeric module graph | Opaque, hard to inspect or extend | +| `browserify-cache.json` stale cache risk | Corrupted cache silently served old module code | +| 20+ npm dependencies just for bundling | Large attack surface, slow installs, difficult version management | + +The new `clay vite` pipeline replaces Browserify/Gulp with **Vite 5 + PostCSS 8**: + +- **Vite** uses Rollup 4 internally for production builds, adding `optimizeDeps` pre-bundling + (esbuild converts CJS `node_modules` before Rollup sees them) and a well-maintained plugin + ecosystem. We use Vite exclusively for its **production build** — the Vite dev server and + HMR are not used. Clay runs a full server-rendered architecture (Amphora) where the browser + never speaks directly to a Vite dev server; watch mode uses Rollup's incremental rebuild + instead. +- PostCSS 8's programmatic API replaces Gulp's stream-based CSS pipeline +- All build steps (JS, CSS, fonts, templates, vendor, media) run **in parallel** +- A human-readable `_manifest.json` replaces the numeric `_registry.json` / `_ids.json` pair +- Watch mode uses Rollup's incremental rebuild — only changed modules are reprocessed +- **Source maps** generated automatically — errors point to exact source file, line, and column +- **Content-hashed filenames** (`chunks/client-A1B2C3.js`) — browsers and CDNs cache files + forever; only changed files get new URLs on deploy +- **Native ESM** output — no custom `window.require()` runtime, browsers handle imports natively +- **Build-time `process.env.NODE_ENV`** — dead branches like `if (dev) {}` are eliminated at + compile time, not runtime +- **`manualChunks` control** — small private modules are inlined into their owner's chunk, + preventing the "hundreds of tiny files" problem that was inherent to Browserify and esbuild +- **Progressive ESM migration** — CJS compatibility shims are named, documented, and have clear + "removed when" conditions; new components can be written as ESM from day one + +## 2. Commands At a Glance + +Both commands coexist. You choose which pipeline to use. + +### Legacy pipeline (Browserify + Gulp) + +```bash +# One-shot compile +clay compile + +# Watch mode +clay compile --watch +``` + +### New pipeline (Vite + PostCSS 8) + +```bash +# One-shot build +clay vite + +# Watch mode +clay vite --watch + +# Minified production build +clay vite --minify +``` + +Both commands read **`claycli.config.js`** in the root of your Clay instance, but they look at +**different config keys** so they never conflict (see [Configuration](#6-configuration)). + +The environment variable `CLAYCLI_VITE_ENABLED=true` enables the Vite pipeline in Dockerfiles and CI. + +## 3. Architecture: Old vs New + +### Old: `clay compile` (Browserify + Gulp) + +``` +clay compile +│ +├── scripts.js ← Browserify megabundler +│ ├── Each component client.js → {name}.client.js (individual file per component) +│ ├── Each component model.js → {name}.model.js + _models-{a-d}.js (alpha-buckets) +│ ├── Each component kiln.js → {name}.kiln.js + _kiln-{a-d}.js (alpha-buckets) +│ ├── Shared deps → {number}.js + _deps-{a-d}.js (alpha-buckets) +│ ├── _prelude.js / _postlude.js ← Browserify custom module runtime (window.require, window.modules) +│ ├── _registry.json ← numeric module ID graph (e.g. { "12": ["4","7"] }) +│ ├── _ids.json ← module ID to filename map +│ └── _client-init.js ← runtime that calls window.require() on each .client module +│ +├── styles.js ← Gulp + PostCSS 7 +│ └── styleguides/**/*.css → public/css/{component}.{styleguide}.css +│ +├── templates.js← Gulp + Handlebars precompile +│ └── components/**/template.hbs → public/js/*.template.js +│ +├── fonts.js ← Gulp copy + CSS concat +│ └── styleguides/*/fonts/* → public/fonts/ + public/css/_linked-fonts.*.css +│ +└── media.js ← Gulp copy + └── components/**/media/* → public/media/ +``` + +**Key runtime behaviour:** `getDependencies()` in view mode walks `_registry.json` for only +the components on the page. `_client-init.js` then calls `window.require(key)` for every +`.client` module in `window.modules` — even if that component's DOM element is absent. + +### New: `clay vite` (Vite + PostCSS 8) + +``` +clay vite +│ +├── scripts.js ← Vite (Rollup 4 internally) +│ ├── View-mode pass (splitting: true): +│ │ ├── Entry: .clay/vite-bootstrap.js ← generated, dynamically imports each client.js +│ │ ├── Entry: .clay/_globals-init.js ← generated, all global/js/*.js in one file +│ │ ├── Shared chunks → public/js/chunks/[name]-[hash].js (content-hashed) +│ │ └── _manifest.json ← human-readable entry → file + chunks map +│ │ +│ └── Kiln-edit pass (inlineDynamicImports: true): +│ ├── Entry: .clay/_kiln-edit-init.js ← generated, registers all model.js + kiln.js +│ └── Single output file: .clay/_kiln-edit-init-[hash].js +│ +├── styles.js ← PostCSS 8 programmatic API (parallel, p-limit 50) +│ └── styleguides/**/*.css → public/css/{component}.{styleguide}.css +│ +├── templates.js← Handlebars precompile (parallel, p-limit 20, progress-tracked) +│ └── components/**/template.hbs → public/js/*.template.js +│ +├── fonts.js ← fs-extra copy + CSS concat +│ └── styleguides/*/fonts/* → public/fonts/ + public/css/_linked-fonts.*.css +│ +├── vendor.js ← fs-extra copy +│ └── clay-kiln/dist/*.js → public/js/ +│ +├── media.js ← fs-extra copy +│ └── components/**/media/* → public/media/ +│ +└── client-env.json ← generated by viteClientEnvPlugin (createClientEnvCollector) + └── collected as a side-effect of the Rollup transform pass — no extra file I/O + (required by amphora-html's addEnvVars() at render time) +``` + +**Key runtime behaviour:** The Vite bootstrap (`_view-init.js`) dynamically imports a +component's `client.js` **only when that component's element exists in the DOM**. +`stickyEvents` in `claycli.config.js` configure a shim that replays one-shot events for +late subscribers (solving the ESM dynamic-import race condition — see Section 8). + +## 4. Pipeline Comparison Diagrams + +Both pipelines share the same source files and produce the same `public/` output. The +differences are in *how* steps are wired, *how* the JS module system works at runtime, and +*how* scripts are resolved and served per page. + +### 4a. Build step execution (sequential vs parallel) + +**🕐 Legacy — `clay compile` (Browserify + Gulp, ~90s)** + +```mermaid +%%{init: {'flowchart': {'nodeSpacing': 60, 'rankSpacing': 70, 'padding': 20}}}%% +flowchart LR + SRC(["📁 Source Files"]):::src + + L1["📦 JS Bundle
Browserify + Babel
30–60 s"]:::slow + L2["🎨 CSS
Gulp + PostCSS 7
15–30 s"]:::slow + L3["📄 Templates
Gulp + Handlebars
10–20 s"]:::med + L4["🔤 Fonts + 🖼 Media
Gulp copy · 2–5 s"]:::fast + + OUT(["📂 public/"]):::out + + SRC --> L1 -->|"waits"| L2 -->|"waits"| L3 -->|"waits"| L4 --> OUT + + classDef src fill:#1e293b,color:#94a3b8,stroke:#334155 + classDef out fill:#1e293b,color:#94a3b8,stroke:#334155 + classDef slow fill:#7f1d1d,color:#fca5a5,stroke:#991b1b + classDef med fill:#78350f,color:#fcd34d,stroke:#92400e + classDef fast fill:#14532d,color:#86efac,stroke:#166534 +``` + +**⚡ New — `clay vite` (Vite + PostCSS 8, ~60s)** + +```mermaid +%%{init: {'flowchart': {'nodeSpacing': 60, 'rankSpacing': 70, 'padding': 20}}}%% +flowchart LR + SRC(["📁 Source Files"]):::src + + N0["🖼 Media
fs-extra · ~0.7 s"]:::fast + N1["📦 JS + Vue
Vite/Rollup · ~30 s"]:::med + N2["🎨 CSS
PostCSS 8 · ~32 s"]:::slow + N3["📄 Templates
Handlebars · ~16 s"]:::med + N4["🔤 Fonts + 📚 Vendor
fs-extra · ~1 s"]:::fast + + OUT(["📂 public/"]):::out + + SRC --> N0 -->|"all at once"| N1 & N2 & N3 & N4 --> OUT + + classDef src fill:#1e293b,color:#94a3b8,stroke:#334155 + classDef out fill:#1e293b,color:#94a3b8,stroke:#334155 + classDef slow fill:#7f1d1d,color:#fca5a5,stroke:#991b1b + classDef med fill:#78350f,color:#fcd34d,stroke:#92400e + classDef fast fill:#14532d,color:#86efac,stroke:#166534 +``` + +**Color guide:** 🔴 slow (>15s) · 🟡 medium (10–20s) · 🟢 fast (<5s) + +| | `clay compile` | `clay vite` | Δ | +|---|---|---|---| +| **Total time** | ~60–120s | ~60s | **~2× faster** | +| **Execution** | Sequential — each step waits | Parallel — all steps run simultaneously after media | ⚠️ Different shape; same end result | +| **JS tool** | Browserify + Babel (megabundles) | Vite (Rollup 4 + manualChunks) | 🔄 Replaced | +| **CSS tool** | Gulp + PostCSS 7 | PostCSS 8 programmatic API | 🔄 Replaced; same plugin ecosystem | +| **Module graph** | `_registry.json` + `_ids.json` | `_manifest.json` (human-readable) | ⚠️ Different format; same purpose | +| **Component loader** | `_client-init.js` — mounts every `.client` module loaded | `vite-bootstrap.js` — mounts only when DOM element present | ✅ Better; avoids dead code execution | +| **JS output** | Per-component files + alpha-bucket dep files | Dynamic import splits + `chunks/` shared deps | ✅ Better; shared deps cached once | + +### 4b. JS module system architecture + +This is the diagram that explains *why* so many other things had to change. The difference in +`resolve-media.js`, `_view-init`, `_kiln-edit-init`, and `_globals-init` all trace back to this. + +**🕐 Legacy — `clay compile` (Browserify runtime module registry)** + +```mermaid +%%{init: {'flowchart': {'nodeSpacing': 60, 'rankSpacing': 70, 'padding': 20}}}%% +flowchart TB + OS["Source files
components/**/client.js · model.js · kiln.js
global/js/*.js"]:::src + + OB["Browserify megabundler + Babel
_prelude.js / _postlude.js
custom window.require runtime"]:::tool + + OR["_registry.json — numeric dep graph
_ids.json — module ID → filename map"]:::artifact + + OI["_client-init.js
calls window.require(key) for every .client
regardless of DOM presence"]:::loader + + OG["_deps-a.js _deps-b.js … (alpha-bucketed shared deps)
_models-a.js _kiln-a.js … (alpha-bucketed edit files)"]:::output + + OS -->|"one big bundle per alpha bucket"| OB + OB -->|"writes"| OR + OB -->|"generates"| OI + OB -->|"outputs"| OG + + classDef src fill:#1e3a5f,color:#93c5fd,stroke:#1d4ed8 + classDef tool fill:#3b1f6e,color:#c4b5fd,stroke:#7c3aed + classDef artifact fill:#422006,color:#fcd34d,stroke:#b45309 + classDef loader fill:#1c2b4a,color:#93c5fd,stroke:#2563eb + classDef output fill:#14532d,color:#86efac,stroke:#166534 +``` + +**⚡ New — `clay vite` (Vite + Rollup static module graph)** + +```mermaid +%%{init: {'flowchart': {'nodeSpacing': 60, 'rankSpacing': 70, 'padding': 20}}}%% +flowchart TB + NS["Source files
components/**/client.js · model.js · kiln.js
global/js/*.js · services/**/*.js"]:::src + + NG["Pre-generated entry points (build-time)
.clay/vite-bootstrap.js
.clay/_kiln-edit-init.js
.clay/_globals-init.js"]:::gen + + NV["Vite build (view pass)
Rollup 4 + manualChunks
splitting: true"]:::tool + + NK["Vite build (kiln pass)
inlineDynamicImports: true
single self-contained output"]:::tool + + NM["_manifest.json
{ '.clay/vite-bootstrap':
{ file: '…-HASH.js', imports: ['chunks/…'] } }"]:::artifact + + NBoot["vite-bootstrap-[hash].js
scans DOM for clay components
dynamic import() only for present elements"]:::loader + + NKiln["_kiln-edit-init-[hash].js
registers all model.js + kiln.js
on window.kiln.componentModels
single file — no splitting"]:::output + + NGL["_globals-init-[hash].js
all global/js/*.js in one file
inlineDynamicImports:true — 1 request"]:::output + + NC["public/js/chunks/
content-hashed shared chunks
cacheable forever"]:::output + + NS -->|"entry points"| NG + NG -->|"feeds"| NV + NG -->|"feeds"| NK + NV -->|"writes"| NM + NV -->|"outputs"| NBoot + NV -->|"extracts shared deps"| NC + NK -->|"outputs"| NKiln + NK -->|"outputs"| NGL + + classDef src fill:#1e3a5f,color:#93c5fd,stroke:#1d4ed8 + classDef gen fill:#1f3b2a,color:#6ee7b7,stroke:#059669 + classDef tool fill:#3b1f6e,color:#c4b5fd,stroke:#7c3aed + classDef artifact fill:#422006,color:#fcd34d,stroke:#b45309 + classDef loader fill:#1c2b4a,color:#93c5fd,stroke:#2563eb + classDef output fill:#14532d,color:#86efac,stroke:#166534 +``` + +**🔁 Same in both pipelines:** CSS (PostCSS plugins → `public/css/`) · Templates (Handlebars precompile → `public/js/*.template.js`) · Fonts (copy + concat → `public/fonts/`) · Media (copy → `public/media/`) + +| Concern | `clay compile` | `clay vite` | Why it matters | +|---|---|---|---| +| **Module registry** | Runtime: `window.modules` built as scripts evaluate | Build-time: `_manifest.json` written once | Old: any code could `window.require()` at any time. New: all wiring is static. | +| **Component mounting** | `_client-init.js` mounts every loaded `.client` module, DOM or not | `vite-bootstrap.js` scans DOM; only imports a component if its element exists | New: no dead component code execution | +| **Edit-mode aggregator** | `window.modules` registry | `_kiln-edit-init.js` pre-registers all model/kiln files on `window.kiln.componentModels` | New: explicit pre-wiring, backward compatible | +| **Shared deps** | Alpha-bucketed dep files (`_deps-a.js`…) — static filenames | Named content-hashed chunks in `public/js/chunks/` | New: unchanged chunks stay cached across deploys | +| **Global scripts** | Individual `` + +// ` +`` +``` + +`omitCacheBusterOnModules` is set to `true` when the `modulepreload: true` option is passed to amphora-html's `setup()`. In `sites/amphora/renderers.js`: + +```javascript +modulepreload: true // enables and strips ?version= from module URLs +``` + +The old Browserify tags are completely unaffected — they still receive `?version=` as before. The change is strictly scoped to `type="module"` and `modulepreload` tags. + +## 15. Bundler Comparison + +For the full technical rationale of why Vite was chosen over esbuild, bare Rollup, and +Browserify — including measured performance data and a pipeline-by-pipeline pros/cons +analysis — see: + +**[`BUNDLER-COMPARISON.md`](./BUNDLER-COMPARISON.md)** + +## 16. Services Pattern and Browser Bundle Hygiene + +The `sites` repo organizes its service layer into three tiers: + +- `services/server/` — Node.js only. May use any server package. +- `services/client/` — Browser only. Communicates with server services over HTTP/REST. +- `services/universal/` — Safe to run in **both** environments. Must not import server-only + packages at module level. + +Clay's Vite pipeline bundles `model.js` files and Kiln plugins for the browser (they run in +Kiln edit mode). Every `require()` in those files — and their entire transitive dependency +chain — ends up in the browser bundle. Any violation of the three-tier contract (e.g. a +`services/universal/` file that imports `clay-log` at the top) silently pulls Node-only +packages into the browser. + +Under the old Browserify pipeline, violations were invisible: Browserify bundled everything +into megabundles and never complained about Node-only packages. Under Vite, unresolvable +Node built-ins produce explicit build errors or runtime crashes. `serviceRewritePlugin` +automatically redirects `services/server/*` to `services/client/*` in browser builds. + +### Why this matters for bundle size + +If `serviceRewritePlugin` is not respected (e.g. a `client.js` file imports +`services/server/query` directly instead of `services/client/query`), the server service's +full dependency tree enters the browser bundle. This can add hundreds of KB of Node-only +transitive dependencies (Elasticsearch clients, `node-fetch`, `iconv-lite` encoding tables, +etc.) that the browser never needs and can never actually use. + +See [`lib/cmd/vite/plugins/service-rewrite.js`](https://github.com/clay/claycli/blob/jordan/yolo-update/lib/cmd/vite/plugins/service-rewrite.js) +for the full implementation and bundle-size impact documentation. + +### Known violations (already fixed) + +| Service | Problem | Fix applied | +|---|---|---| +| `services/universal/styles.js` | Imported full PostCSS toolchain (~666 KB gz) | Moved to `services/server/styles.js`; added no-op `services/client/styles.js` stub | +| `services/universal/coral.js` | Imported `node-fetch` + `iconv-lite` (~576 KB gz) | Moved to `services/server/coral.js`; added `services/client/coral.js` stub with only browser-safe exports | +| `services/universal/log.js` | Imported `clay-log` (Node-only) at top level | Moved `clay-log` import inside lazy conditional branches | diff --git a/cli/import.js b/cli/import.js index 45b6b9dd..a7abd5d3 100644 --- a/cli/import.js +++ b/cli/import.js @@ -2,6 +2,8 @@ const _ = require('lodash'), pluralize = require('pluralize'), chalk = require('chalk'), + b64 = require('base-64'), + nodeUrl = require('url'), options = require('./cli-options'), reporter = require('../lib/reporters'), importItems = require('../lib/cmd/import'); @@ -45,10 +47,21 @@ function handler(argv) { } }) .toArray((results) => { - const pages = _.map(_.filter(results, (result) => result.type === 'success' && _.includes(result.message, 'pages')), (page) => `${page.message}.html`); + const protocol = nodeUrl.parse(argv.url).protocol || 'https:', + pages = _.map(_.filter(results, (result) => result.type === 'success' && _.includes(result.message, 'pages')), (page) => `${page.message}.html`), + publicUrls = _.map( + _.filter(results, (result) => result.type === 'success' && _.includes(result.message, '_uris/')), + (result) => { + const encoded = result.message.split('/_uris/')[1]; + + return `${protocol}//${b64.decode(encoded)}`; + } + ); reporter.logSummary(argv.reporter, 'import', (successes) => { - if (successes && pages.length) { + if (successes && pages.length && publicUrls.length) { + return { success: true, message: `Imported ${pluralize('page', pages.length, true)}\n${chalk.gray(pages.join('\n'))}\n${chalk.cyan('Public URL(s):\n' + publicUrls.join('\n'))}` }; + } else if (successes && pages.length) { return { success: true, message: `Imported ${pluralize('page', pages.length, true)}\n${chalk.gray(pages.join('\n'))}` }; } else if (successes) { return { success: true, message: `Imported ${pluralize('uri', successes, true)}` }; diff --git a/cli/index.js b/cli/index.js index 54e203e1..abf49e27 100755 --- a/cli/index.js +++ b/cli/index.js @@ -21,7 +21,9 @@ const yargs = require('yargs'), e: 'export', i: 'import', l: 'lint', - p: 'pack' + p: 'pack', + v: 'vite', + vite: 'vite', }, listCommands = Object.keys(commands).concat(Object.values(commands)); diff --git a/cli/vite.js b/cli/vite.js new file mode 100644 index 00000000..c25664cb --- /dev/null +++ b/cli/vite.js @@ -0,0 +1,79 @@ +'use strict'; + +const { build, watch } = require('../lib/cmd/vite'); +const log = require('./log').setup({ file: __filename }); + +function builder(yargs) { + return yargs + .usage('Usage: $0 [options]') + .option('watch', { + alias: 'w', + type: 'boolean', + description: 'Watch for file changes and rebuild automatically', + default: false, + }) + .option('minify', { + alias: 'm', + type: 'boolean', + description: 'Minify output (also enabled by CLAYCLI_COMPILE_MINIFIED env var)', + default: !!process.env.CLAYCLI_COMPILE_MINIFIED, + }) + .option('entry', { + alias: 'e', + type: 'array', + description: 'Additional entry-point file paths (supplements the default component globs)', + default: [], + }) + .example('$0', 'Build all component scripts and assets with Vite') + .example('$0 --watch', 'Rebuild on every file change') + .example('$0 --minify', 'Build and minify for production'); +} + +async function handler(argv) { + const options = { + minify: argv.minify, + extraEntries: argv.entry || [], + }; + + if (argv.watch) { + try { + const ctx = await watch({ + ...options, + // Called once after the first successful build — correct place for the + // "ready" message because watch() resolves before BUNDLE_END fires. + onReady() { + log('info', 'Watching for changes — press Ctrl+C to stop'); + }, + // Only report errors here; successful rebuilds are already logged by + // scripts.js with the module-count suffix to avoid duplicate output. + onRebuild(errors) { + errors.forEach(e => log('error', e.message || String(e))); + }, + }); + + process.on('SIGINT', () => { + ctx.dispose().then(() => process.exit(0)); + }); + + process.on('SIGTERM', () => { + ctx.dispose().then(() => process.exit(0)); + }); + } catch (error) { + log('error', 'Watch setup failed', { error: error.message }); + process.exit(1); + } + } else { + try { + await build(options); + } catch (error) { + log('error', 'Build failed', { error: error.message }); + process.exit(1); + } + } +} + +exports.aliases = []; +exports.builder = builder; +exports.command = 'vite'; +exports.describe = 'Compile component scripts and assets with Vite (Rollup production, HMR watch)'; +exports.handler = handler; diff --git a/index.js b/index.js index 8e10dc3b..f7920a7b 100644 --- a/index.js +++ b/index.js @@ -7,6 +7,7 @@ module.exports.lint = require('./lib/cmd/lint'); module.exports.import = require('./lib/cmd/import'); module.exports.export = require('./lib/cmd/export'); module.exports.compile = require('./lib/cmd/compile'); +module.exports.vite = require('./lib/cmd/vite'); module.exports.gulp = require('gulp'); // A reference to the Gulp instance so that external tasks can reference a common package module.exports.mountComponentModules = require('./lib/cmd/pack/mount-component-modules'); module.exports.getWebpackConfig = require('./lib/cmd/pack/get-webpack-config'); diff --git a/lib/cmd/compile/fonts.js b/lib/cmd/compile/fonts.js index e25502b3..66551d39 100644 --- a/lib/cmd/compile/fonts.js +++ b/lib/cmd/compile/fonts.js @@ -3,7 +3,8 @@ const _ = require('lodash'), h = require('highland'), afs = require('amphora-fs'), path = require('path'), - es = require('event-stream'), + through2 = require('through2'), + mergeStream = require('merge-stream'), gulp = require('gulp'), newer = require('../../gulp-plugins/gulp-newer'), concat = require('gulp-concat'), @@ -123,7 +124,7 @@ function getFontCSS(file, styleguide, isInlined) { css += `src: url(${assetHost}${assetPath}/fonts/${styleguide}/${fileName}); }`; } - file.contents = new Buffer(css); + file.contents = Buffer.from(css, 'utf8'); return file; } } @@ -160,11 +161,11 @@ function compile(options = {}) { inlinedFontsTask = gulp.src(fontsSrc) // if a font in the styleguide is changed, recompile the result file .pipe(newer({ dest: path.join(destPath, 'css', `_inlined-fonts.${styleguide}.css`), ctime: true })) - .pipe(es.mapSync((file) => getFontCSS(file, styleguide, true))) + .pipe(through2.obj((file, enc, cb) => cb(null, getFontCSS(file, styleguide, true)))) .pipe(concat(`_inlined-fonts.${styleguide}.css`)) .pipe(gulpIf(Boolean(minify), cssmin())) .pipe(gulp.dest(path.join(destPath, 'css'))) - .pipe(es.mapSync((file) => ({ type: 'success', message: file.path }))); + .pipe(through2.obj((file, enc, cb) => cb(null, { type: 'success', message: file.path }))); streams.push(inlinedFontsTask); } @@ -176,18 +177,18 @@ function compile(options = {}) { // copy font file itself (to public/fonts//) .pipe(rename({ dirname: styleguide })) .pipe(gulp.dest(path.join(destPath, 'fonts'))) - .pipe(es.mapSync((file) => getFontCSS(file, styleguide, false))) + .pipe(through2.obj((file, enc, cb) => cb(null, getFontCSS(file, styleguide, false)))) .pipe(concat(`_linked-fonts.${styleguide}.css`)) .pipe(gulpIf(Boolean(minify), cssmin())) .pipe(gulp.dest(path.join(destPath, 'css'))) - .pipe(es.mapSync((file) => ({ type: 'success', message: file.path }))); + .pipe(through2.obj((file, enc, cb) => cb(null, { type: 'success', message: file.path }))); streams.push(linkedFontsTask); } return streams; }, []); - return es.merge(tasks); + return mergeStream(...tasks); } gulp.task('fonts', () => { diff --git a/lib/cmd/compile/get-script-dependencies.js b/lib/cmd/compile/get-script-dependencies.js index afc80d4b..e96e9e7f 100644 --- a/lib/cmd/compile/get-script-dependencies.js +++ b/lib/cmd/compile/get-script-dependencies.js @@ -1,7 +1,7 @@ 'use strict'; const _ = require('lodash'), path = require('path'), - glob = require('glob'), + { globSync } = require('glob'), // destination paths destPath = path.resolve(process.cwd(), 'public', 'js'), registryPath = path.resolve(destPath, '_registry.json'); @@ -14,7 +14,7 @@ const _ = require('lodash'), function getAllDeps(minify) { const fileName = minify ? '_deps-?-?.js' : '+([0-9]).js'; - return glob.sync(path.join(destPath, fileName)).map((filepath) => path.parse(filepath).name); + return globSync(path.join(destPath, fileName)).map((filepath) => path.parse(filepath).name); } @@ -27,7 +27,7 @@ function getAllDeps(minify) { function getAllModels(minify) { const fileName = minify ? '_models-?-?.js' : '*.model.js'; - return glob.sync(path.join(destPath, fileName)).map((filepath) => path.parse(filepath).name); + return globSync(path.join(destPath, fileName)).map((filepath) => path.parse(filepath).name); } /** @@ -38,7 +38,7 @@ function getAllModels(minify) { function getAllKilnjs(minify) { const fileName = minify ? '_kiln-?-?.js' : '*.kiln.js'; - return glob.sync(path.join(destPath, fileName)).map((filepath) => path.parse(filepath).name); + return globSync(path.join(destPath, fileName)).map((filepath) => path.parse(filepath).name); } /** @@ -49,7 +49,7 @@ function getAllKilnjs(minify) { function getAllTemplates(minify) { const fileName = minify ? '_templates-?-?.js' : '*.template.js'; - return glob.sync(path.join(destPath, fileName)).map((filepath) => path.parse(filepath).name); + return globSync(path.join(destPath, fileName)).map((filepath) => path.parse(filepath).name); } /** diff --git a/lib/cmd/compile/media.js b/lib/cmd/compile/media.js index cd108ea4..0213729e 100644 --- a/lib/cmd/compile/media.js +++ b/lib/cmd/compile/media.js @@ -6,7 +6,8 @@ const _ = require('lodash'), gulp = require('gulp'), rename = require('gulp-rename'), changed = require('gulp-changed'), - es = require('event-stream'), + through2 = require('through2'), + mergeStream = require('merge-stream'), reporters = require('../../reporters'), destPath = path.join(process.cwd(), 'public', 'media'), mediaGlobs = '*.+(jpg|jpeg|png|gif|webp|svg|ico)'; @@ -58,31 +59,31 @@ function compile(options = {}) { .pipe(rename({ dirname: path.join('components', component.name) })) .pipe(changed(destPath)) .pipe(gulp.dest(destPath)) - .pipe(es.mapSync((file) => ({ type: 'success', message: file.path }))); + .pipe(through2.obj((file, enc, cb) => cb(null, { type: 'success', message: file.path }))); }), layoutsTask = _.map(layoutsSrc, (layout) => { return gulp.src(layout.path) .pipe(rename({ dirname: path.join('layouts', layout.name) })) .pipe(changed(destPath)) .pipe(gulp.dest(destPath)) - .pipe(es.mapSync((file) => ({ type: 'success', message: file.path }))); + .pipe(through2.obj((file, enc, cb) => cb(null, { type: 'success', message: file.path }))); }), styleguidesTask = _.map(styleguidesSrc, (styleguide) => { return gulp.src(styleguide.path) .pipe(rename({ dirname: path.join('styleguides', styleguide.name) })) .pipe(changed(destPath)) .pipe(gulp.dest(destPath)) - .pipe(es.mapSync((file) => ({ type: 'success', message: file.path }))); + .pipe(through2.obj((file, enc, cb) => cb(null, { type: 'success', message: file.path }))); }), sitesTask = _.map(sitesSrc, (site) => { return gulp.src(site.path) .pipe(rename({ dirname: path.join('sites', site.name) })) .pipe(changed(destPath)) .pipe(gulp.dest(destPath)) - .pipe(es.mapSync((file) => ({ type: 'success', message: file.path }))); + .pipe(through2.obj((file, enc, cb) => cb(null, { type: 'success', message: file.path }))); }); - return es.merge(componentTasks.concat(layoutsTask, styleguidesTask, sitesTask)); + return mergeStream(...componentTasks.concat(layoutsTask, styleguidesTask, sitesTask)); } gulp.task('media', () => { diff --git a/lib/cmd/compile/scripts.js b/lib/cmd/compile/scripts.js index 1a134dad..e7572862 100644 --- a/lib/cmd/compile/scripts.js +++ b/lib/cmd/compile/scripts.js @@ -3,13 +3,12 @@ const _ = require('lodash'), fs = require('fs-extra'), path = require('path'), h = require('highland'), - glob = require('glob'), + { globSync } = require('glob'), chokidar = require('chokidar'), // gulp, gulp plugins, and deps gulp = require('gulp'), changed = require('gulp-changed'), replace = require('gulp-replace'), - es = require('event-stream'), // browserify / megabundler deps browserify = require('browserify'), browserifyCache = require('browserify-cache-api'), @@ -19,7 +18,6 @@ const _ = require('lodash'), browserifyExtractRegistry = require('browserify-extract-registry'), browserifyExtractIds = require('browserify-extract-ids'), browserifyGlobalPack = require('browserify-global-pack'), - bundleCollapser = require('bundle-collapser/plugin'), transformTools = require('browserify-transform-tools'), unreachableCodeTransform = require('unreachable-branch-transform'), vueify = require('@nymag/vueify'), @@ -78,7 +76,7 @@ function buildKiln() { return h(gulp.src(kilnGlob) .pipe(changed(destPath, { hasChanged: helpers.hasChanged })) .pipe(gulp.dest(destPath)) - .pipe(es.mapSync((file) => ({ type: 'success', message: file.path })))); + .pipe(through2.obj((file, enc, cb) => cb(null, { type: 'success', message: file.path })))); } /** @@ -93,7 +91,7 @@ function copyClientInit() { .pipe(babel(babelConfig)) .pipe(replace('#NODE_ENV#', process.env.NODE_ENV || '')) .pipe(gulp.dest(destPath)) - .pipe(es.mapSync((file) => ({ type: 'success', message: file.path })))); + .pipe(through2.obj((file, enc, cb) => cb(null, { type: 'success', message: file.path })))); } /** @@ -396,9 +394,13 @@ function buildScripts(entries, options = {}) { .plugin(browserifyGlobalPack, { getOutfile, cache: options.cache.path - }) - // shorten bundle size by rewriting require() to use module IDs - .plugin(bundleCollapser); + }); + // bundle-collapser was removed to keep this pipeline functional during the per-site Vite rollout + // (CLAYCLI_VITE_ENABLED / CLAYCLI_VITE_SITES). Its bundled falafel/acorn (acorn 7.x) cannot parse + // ES2021+ syntax (logical assignment &&=, ||=, ??= etc.) present in modern npm dependencies, + // causing a hard build failure. It was a size-only optimization (replacing full require() path + // strings with integer IDs) whose savings are negligible after minification + gzip compression. + // Once all sites have migrated to Vite, this entire pipeline can be retired. if (options.minify) { bundler.transform(uglifyify, { global: true, output: { inline_script: true }}); @@ -440,14 +442,14 @@ function compile(options = {}) { minify = options.minify || variables.minify || false, globs = options.globs || [], reporter = options.reporter || 'pretty', - globFiles = globs.length ? _.flatten(_.map(globs, (g) => glob.sync(path.join(process.cwd(), g)))) : [], + globFiles = globs.length ? _.flatten(_.map(globs, (g) => globSync(path.join(process.cwd(), g)))) : [], // client.js, model.js, kiln plugins, and legacy global scripts are passed to megabundler - bundleEntries = glob.sync(componentClientsGlob).concat( - glob.sync(componentModelsGlob), - glob.sync(componentKilnGlob), - glob.sync(layoutClientsGlob), - glob.sync(layoutModelsGlob), - glob.sync(kilnPluginsGlob), + bundleEntries = globSync(componentClientsGlob).concat( + globSync(componentModelsGlob), + globSync(componentKilnGlob), + globSync(layoutClientsGlob), + globSync(layoutModelsGlob), + globSync(kilnPluginsGlob), globFiles ), // options are set beforehand, so we can grab the cached files to watch afterwards diff --git a/lib/cmd/compile/styles.js b/lib/cmd/compile/styles.js index f70059a7..e835805a 100644 --- a/lib/cmd/compile/styles.js +++ b/lib/cmd/compile/styles.js @@ -3,7 +3,7 @@ const _ = require('lodash'), fs = require('fs-extra'), h = require('highland'), path = require('path'), - es = require('event-stream'), + through2 = require('through2'), gulp = require('gulp'), rename = require('gulp-rename'), changed = require('gulp-changed'), @@ -128,7 +128,7 @@ function compile(options = {}) { ].concat(plugins))) .pipe(gulpIf(Boolean(minify), cssmin())) .pipe(gulp.dest(destPath)) - .pipe(es.mapSync((file) => ({ type: 'success', message: path.basename(file.path) }))); + .pipe(through2.obj((file, enc, cb) => cb(null, { type: 'success', message: path.basename(file.path) }))); } gulp.task('styles', () => { diff --git a/lib/cmd/compile/templates.js b/lib/cmd/compile/templates.js index fcf0bfc4..2c19cb70 100644 --- a/lib/cmd/compile/templates.js +++ b/lib/cmd/compile/templates.js @@ -8,7 +8,7 @@ const _ = require('lodash'), gulp = require('gulp'), rename = require('gulp-rename'), groupConcat = require('gulp-group-concat'), - es = require('event-stream'), + through2 = require('through2'), clayHbs = require('clayhandlebars'), hbs = clayHbs(), uglify = require('uglify-js'), @@ -109,7 +109,7 @@ function inlineRead(source, filepath) { function wrapTemplate(file) { let source = _.includes(file.path, 'clay-kiln') ? file.contents.toString('utf8') : inlineRead(file.contents.toString('utf8'), file.path); - file.contents = new Buffer(clayHbs.wrapPartial(_.last(path.dirname(file.path).split(path.sep)), source)); + file.contents = Buffer.from(clayHbs.wrapPartial(_.last(path.dirname(file.path).split(path.sep)), source), 'utf8'); return file; } @@ -122,7 +122,7 @@ function precompile(file) { const name = path.parse(file.path).name.replace('.template', ''); try { - file.contents = new Buffer(hbs.precompile(file.contents.toString('utf8'))); + file.contents = Buffer.from(hbs.precompile(file.contents.toString('utf8')), 'utf8'); return file; } catch (e) { console.log(chalk.red(`Error precompiling template "${name}": `) + e.message); @@ -139,7 +139,7 @@ function registerTemplate(file) { const name = path.parse(file.path).name.replace('.template', ''), contents = file.contents.toString('utf8'); - file.contents = new Buffer(`window.kiln.componentTemplates['${name}']=${contents}\n`); + file.contents = Buffer.from(`window.kiln.componentTemplates['${name}']=${contents}\n`, 'utf8'); return file; } @@ -158,7 +158,7 @@ function minifyTemplate(file, shouldMinify) { try { const minified = uglify.minify(file.contents.toString('utf8'), { output: { inline_script: true } }); - file.contents = new Buffer(minified.code); + file.contents = Buffer.from(minified.code, 'utf8'); return file; } catch (e) { const name = path.parse(file.path).name.replace('.template', ''); @@ -185,7 +185,7 @@ function compile(options = {}) { reporter = options.reporter || 'pretty'; function concatTemplates() { - return minify ? groupConcat(bundles) : es.mapSync((file) => file); + return minify ? groupConcat(bundles) : through2.obj((file, enc, cb) => cb(null, file)); } function buildPipeline() { @@ -216,18 +216,18 @@ function compile(options = {}) { hasChanged: hasTemplateChanged(minify) })) */ - .pipe(es.mapSync(wrapTemplate)) - .pipe(es.mapSync(precompile)) + .pipe(through2.obj((file, enc, cb) => cb(null, wrapTemplate(file)))) + .pipe(through2.obj((file, enc, cb) => cb(null, precompile(file)))) .on('error', (err) => { if (!watch) { throw err; } }) - .pipe(es.mapSync(registerTemplate)) - .pipe(es.mapSync((file) => minifyTemplate(file, minify))) + .pipe(through2.obj((file, enc, cb) => cb(null, registerTemplate(file)))) + .pipe(through2.obj((file, enc, cb) => cb(null, minifyTemplate(file, minify)))) .pipe(concatTemplates()) // when minifying, concat to '_templates--.js' .pipe(gulp.dest(destPath)) - .pipe(es.mapSync((file) => ({ type: 'success', message: file.path }))); + .pipe(through2.obj((file, enc, cb) => cb(null, { type: 'success', message: file.path }))); } gulp.task('templates', () => { diff --git a/lib/cmd/import.test.js b/lib/cmd/import.test.js index f2c163e6..2e045268 100644 --- a/lib/cmd/import.test.js +++ b/lib/cmd/import.test.js @@ -27,7 +27,9 @@ describe('import', () => { it('returns error if bad JSON', () => { return lib(']{"a"\:}', url).toPromise(Promise).then((res) => { - expect(res).toEqual({ type: 'error', message: 'JSON syntax error: Unexpected token ] in JSON at position 0', details: ']{"a":}' }); + expect(res.type).toBe('error'); + expect(res.message).toMatch(/^JSON syntax error:/); + expect(res.details).toBe(']{"a":}'); }); }); @@ -246,7 +248,7 @@ describe('import', () => { it('returns error if bad json', () => { return fn(JSON.stringify({ a: 'hi' }) + 'a', url).toPromise(Promise).catch((e) => { - expect(e.message).toBe('JSON parser error: Unexpected token a in JSON at position 10'); + expect(e.message).toMatch(/^JSON parser error:/); }); }); diff --git a/lib/cmd/vite/fonts.js b/lib/cmd/vite/fonts.js new file mode 100644 index 00000000..79e2ee78 --- /dev/null +++ b/lib/cmd/vite/fonts.js @@ -0,0 +1,106 @@ +'use strict'; + +const path = require('path'); +const fs = require('fs-extra'); +const { globSync } = require('glob'); + +const CWD = process.cwd(); + +const FONT_EXTS = 'css,woff,woff2,otf,ttf'; +const FONTS_SRC_GLOB = path.join(CWD, 'styleguides', '*', 'fonts', `*.{${FONT_EXTS}}`); + +// Output destinations +const CSS_DEST = path.join(CWD, 'public', 'css'); +const BINARY_DEST = path.join(CWD, 'public', 'fonts'); + +const ASSET_HOST = process.env.CLAYCLI_COMPILE_ASSET_HOST + ? process.env.CLAYCLI_COMPILE_ASSET_HOST.replace(/\/$/, '') + : ''; +const ASSET_PATH = process.env.CLAYCLI_COMPILE_ASSET_PATH || ''; + +/** + * Extract the styleguide name from an absolute font file path. + * Expected structure: .../styleguides/{sg}/fonts/{file} + * + * @param {string} srcPath + * @returns {string} styleguide name + */ +function getStyleguide(srcPath) { + const parts = srcPath.split(path.sep); + const sgIdx = parts.lastIndexOf('styleguides'); + + return parts[sgIdx + 1]; +} + +/** + * Process fonts: + * - CSS files: apply $asset-host / $asset-path substitution, then concatenate + * all per-styleguide CSS into public/css/_linked-fonts.{sg}.css so that + * amphora-html can find and inline the @font-face declarations. + * - Binary files (.woff, .woff2, .otf, .ttf): copy as-is to + * public/fonts/{styleguide}/ for cases where fonts are self-hosted. + * + * @returns {Promise} count of files processed + */ +async function buildFonts() { + const files = globSync(FONTS_SRC_GLOB); + + if (files.length === 0) return 0; + + // Group by styleguide + const byStyleguide = {}; + + for (const srcPath of files) { + const sg = getStyleguide(srcPath); + + if (!byStyleguide[sg]) { + byStyleguide[sg] = { css: [], binary: [] }; + } + + const ext = path.extname(srcPath).toLowerCase(); + + if (ext === '.css') { + byStyleguide[sg].css.push(srcPath); + } else { + byStyleguide[sg].binary.push(srcPath); + } + } + + await Promise.all([ + fs.ensureDir(CSS_DEST), + fs.ensureDir(BINARY_DEST), + ]); + + await Promise.all(Object.entries(byStyleguide).map(async ([sg, { css, binary }]) => { + // Write _linked-fonts.{sg}.css — amphora-html looks for this file to inject + // @font-face declarations into every page that uses this styleguide. + if (css.length > 0) { + const cssChunks = await Promise.all(css.map(async (srcPath) => { + let content = await fs.readFile(srcPath, 'utf8'); + + content = content.replace(/\$asset-host/g, ASSET_HOST); + content = content.replace(/\$asset-path/g, ASSET_PATH); + + return content; + })); + + await fs.writeFile( + path.join(CSS_DEST, `_linked-fonts.${sg}.css`), + cssChunks.join('\n'), + 'utf8' + ); + } + + // Copy binary font files (self-hosted scenarios) + await Promise.all(binary.map(async (srcPath) => { + const destPath = path.join(BINARY_DEST, sg, path.basename(srcPath)); + + await fs.ensureDir(path.dirname(destPath)); + await fs.copy(srcPath, destPath, { overwrite: true }); + })); + })); + + return files.length; +} + +module.exports = { buildFonts, FONTS_SRC_GLOB }; diff --git a/lib/cmd/vite/fonts.test.js b/lib/cmd/vite/fonts.test.js new file mode 100644 index 00000000..be70f68f --- /dev/null +++ b/lib/cmd/vite/fonts.test.js @@ -0,0 +1,173 @@ +/* global jest:false */ +'use strict'; + +const path = require('path'); +const fs = require('fs-extra'); +const os = require('os'); + +let tmpDir; + +let originalCwd; + +beforeEach(async () => { + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'claycli-fonts-')); + originalCwd = process.cwd; + process.cwd = () => tmpDir; + + // Create representative font source tree across two styleguides + const defaultFonts = path.join(tmpDir, 'styleguides', '_default', 'fonts'); + const mobileFonts = path.join(tmpDir, 'styleguides', 'mobile', 'fonts'); + + await fs.ensureDir(defaultFonts); + await fs.ensureDir(mobileFonts); + + await fs.writeFile( + path.join(defaultFonts, 'fonts.css'), + '@font-face { src: url($asset-host$asset-path/fonts/Gotham.woff2); }' + ); + await fs.writeFile(path.join(defaultFonts, 'Gotham.woff2'), 'woff2-binary'); + await fs.writeFile(path.join(defaultFonts, 'Gotham.ttf'), 'ttf-binary'); + await fs.writeFile( + path.join(mobileFonts, 'fonts.css'), + '@font-face { src: url($asset-host$asset-path/fonts/Roboto.woff); }' + ); +}); + +afterEach(async () => { + process.cwd = originalCwd; + delete process.env.CLAYCLI_COMPILE_ASSET_HOST; + delete process.env.CLAYCLI_COMPILE_ASSET_PATH; + await fs.remove(tmpDir); + jest.resetModules(); +}); + +describe('buildFonts', () => { + it('returns 0 when no font files exist', async () => { + await fs.remove(path.join(tmpDir, 'styleguides')); + + const { buildFonts } = require('./fonts'); + const count = await buildFonts(); + + expect(count).toBe(0); + }); + + it('returns total count of all font files processed', async () => { + const { buildFonts } = require('./fonts'); + const count = await buildFonts(); + + // 1 css + 2 binary for _default, 1 css for mobile = 4 + expect(count).toBe(4); + }); + + it('writes _linked-fonts.{sg}.css to public/css/', async () => { + const { buildFonts } = require('./fonts'); + + await buildFonts(); + + const defaultOut = path.join(tmpDir, 'public', 'css', '_linked-fonts._default.css'); + const mobileOut = path.join(tmpDir, 'public', 'css', '_linked-fonts.mobile.css'); + + expect(fs.existsSync(defaultOut)).toBe(true); + expect(fs.existsSync(mobileOut)).toBe(true); + }); + + it('copies binary font files to public/fonts/{sg}/', async () => { + const { buildFonts } = require('./fonts'); + + await buildFonts(); + + const woff2 = path.join(tmpDir, 'public', 'fonts', '_default', 'Gotham.woff2'); + const ttf = path.join(tmpDir, 'public', 'fonts', '_default', 'Gotham.ttf'); + + expect(fs.existsSync(woff2)).toBe(true); + expect(fs.existsSync(ttf)).toBe(true); + }); + + it('substitutes $asset-host in CSS output', async () => { + process.env.CLAYCLI_COMPILE_ASSET_HOST = 'https://cdn.example.com'; + + const { buildFonts } = require('./fonts'); + + await buildFonts(); + + const content = await fs.readFile( + path.join(tmpDir, 'public', 'css', '_linked-fonts._default.css'), + 'utf8' + ); + + expect(content).toContain('https://cdn.example.com'); + expect(content).not.toContain('$asset-host'); + }); + + it('substitutes $asset-path in CSS output', async () => { + process.env.CLAYCLI_COMPILE_ASSET_PATH = '/static/v2'; + + const { buildFonts } = require('./fonts'); + + await buildFonts(); + + const content = await fs.readFile( + path.join(tmpDir, 'public', 'css', '_linked-fonts._default.css'), + 'utf8' + ); + + expect(content).toContain('/static/v2'); + expect(content).not.toContain('$asset-path'); + }); + + it('replaces both $asset-host and $asset-path together', async () => { + process.env.CLAYCLI_COMPILE_ASSET_HOST = 'https://cdn.example.com'; + process.env.CLAYCLI_COMPILE_ASSET_PATH = '/v3'; + + const { buildFonts } = require('./fonts'); + + await buildFonts(); + + const content = await fs.readFile( + path.join(tmpDir, 'public', 'css', '_linked-fonts._default.css'), + 'utf8' + ); + + expect(content).toContain('https://cdn.example.com/v3'); + expect(content).not.toContain('$asset-host'); + expect(content).not.toContain('$asset-path'); + }); + + it('strips trailing slash from CLAYCLI_COMPILE_ASSET_HOST', async () => { + process.env.CLAYCLI_COMPILE_ASSET_HOST = 'https://cdn.example.com/'; + + const { buildFonts } = require('./fonts'); + + await buildFonts(); + + const content = await fs.readFile( + path.join(tmpDir, 'public', 'css', '_linked-fonts._default.css'), + 'utf8' + ); + + expect(content).not.toContain('https://cdn.example.com//'); + expect(content).toContain('https://cdn.example.com'); + }); + + it('creates public/css and public/fonts directories if they do not exist', async () => { + const { buildFonts } = require('./fonts'); + + await buildFonts(); + + expect(fs.existsSync(path.join(tmpDir, 'public', 'css'))).toBe(true); + expect(fs.existsSync(path.join(tmpDir, 'public', 'fonts'))).toBe(true); + }); + + it('handles styleguides that have only CSS files (no binaries)', async () => { + const { buildFonts } = require('./fonts'); + + // mobile styleguide only has fonts.css, no binary files + await buildFonts(); + + const mobileOut = path.join(tmpDir, 'public', 'css', '_linked-fonts.mobile.css'); + + expect(fs.existsSync(mobileOut)).toBe(true); + // No binary fonts directory created for mobile + expect(fs.existsSync(path.join(tmpDir, 'public', 'fonts', 'mobile'))).toBe(false); + }); +}); diff --git a/lib/cmd/vite/generate-bootstrap.js b/lib/cmd/vite/generate-bootstrap.js new file mode 100644 index 00000000..68771b7e --- /dev/null +++ b/lib/cmd/vite/generate-bootstrap.js @@ -0,0 +1,182 @@ +'use strict'; + +const fs = require('fs-extra'); +const path = require('path'); +const { globSync } = require('glob'); +const { getConfigValue } = require('../../config-file-helpers'); + +const CWD = process.cwd(); +const CLAY_DIR = path.join(CWD, '.clay'); +const VITE_BOOTSTRAP_FILE = path.join(CLAY_DIR, 'vite-bootstrap.js'); +const GLOBALS_INIT_FILE = path.join(CLAY_DIR, '_globals-init.js'); + +const VITE_BOOTSTRAP_KEY = '.clay/vite-bootstrap'; + +/** + * Component mount runtime injected into the bootstrap. + * + * Scans DOM comments for Clay component markers, pre-loads all matched + * client modules in parallel, then walks [data-uri] elements and mounts + * each component via its default export or via DS.get(). + */ +const MOUNT_RUNTIME = `\ +// ── Component mounting (Vite bootstrap) ───────────────────────────────────── +var CLAY_INSTANCE_KIND = /_components\\/(.+?)(\\/instances|$)/; + +function mountComponentModules() { + performance.mark('clay-components-start'); + + return new Promise(function(resolve) { + var iterator = document.createNodeIterator( + document.documentElement, + NodeFilter.SHOW_COMMENT, + function(node) { + return node.nodeValue && node.nodeValue.indexOf('_components/') !== -1 + ? NodeFilter.FILTER_ACCEPT + : NodeFilter.FILTER_SKIP; + } + ); + + var node, preloads = []; + + while ((node = iterator.nextNode())) { + var pm = node.nodeValue.match(CLAY_INSTANCE_KIND); + + if (pm) { + var preloadKey = 'components/' + pm[1] + '/client.js'; + + if (_clayClientModules[preloadKey]) { + preloads.push(_clayClientModules[preloadKey]()); + } + } + } + + resolve(Promise.allSettled(preloads)); + }).then(function() { + var els = Array.from(document.querySelectorAll('[data-uri*="_components/"]')); + var mounted = 0, errors = []; + + return Promise.allSettled(els.map(function(el) { + var m = CLAY_INSTANCE_KIND.exec(el.dataset.uri); + + if (!m) return Promise.resolve(); + + var name = m[1]; + var key = 'components/' + name + '/client.js'; + var loader = _clayClientModules[key]; + + if (!loader) return Promise.resolve(); + + return loader() + .then(function(mod) { return mod.default != null ? mod.default : mod; }) + .then(function(mod) { + if (typeof mod === 'function') { + try { mod(el); mounted++; } catch(e) { errors.push(name + ' (mount): ' + (e && e.message || e)); } + } else if (window.DS && typeof window.DS.get === 'function') { + try { window.DS.get(name, el); mounted++; } catch(e) { errors.push(name + ': ' + e.message); } + } + }) + .catch(function(e) { errors.push(name + ' (load): ' + (e && e.message || e)); }); + })).then(function() { + console.debug('[clay vite] mounted ' + mounted + '/' + els.length + ' components'); + if (errors.length) console.warn('[clay vite] mount errors:', errors); + }); + }).finally(function() { + performance.mark('clay-components-end'); + performance.measure('clay-components', 'clay-components-start', 'clay-components-end'); + var dur = (performance.getEntriesByName('clay-components').pop() || {}).duration; + + console.debug('[clay vite] components took ' + dur + 'ms'); + }); +} + +mountComponentModules().catch(console.error); +`; + +/** + * Generate .clay/vite-bootstrap.js — the single ESM entry point for view mode. + * + * Contains: + * 1. Static import of _globals-init.js (runs synchronously before any + * component dynamic import, ensuring window.DS etc. are available). + * 2. Sticky custom-event shim (when stickyEvents is configured). + * 3. _clayClientModules map — one lazy import() per component/layout client.js. + * 4. mountComponentModules() runtime — scans DOM and mounts components. + * + * @returns {Promise} absolute path to the written bootstrap file + */ +async function generateViteBootstrap() { + const clientFiles = [ + ...globSync(path.join(CWD, 'components', '**', 'client.js')), + ...globSync(path.join(CWD, 'layouts', '**', 'client.js')), + ]; + + const toRel = absPath => { + const rel = path.relative(CLAY_DIR, absPath).replace(/\\/g, '/'); + + return rel.startsWith('.') ? rel : `./${rel}`; + }; + + const moduleEntries = clientFiles.map(f => { + const key = path.relative(CWD, f).replace(/\\/g, '/'); + + return ` ${JSON.stringify(key)}: () => import(${JSON.stringify(toRel(f))})`; + }).join(',\n'); + + // ── Sticky events shim ─────────────────────────────────────────────────── + const stickyEvents = getConfigValue('stickyEvents') || []; + const stickyListeners = stickyEvents + .map(n => ` _orig(${JSON.stringify(n)}, function(ev) { fired[${JSON.stringify(n)}] = ev.detail; });`) + .join('\n'); + + const stickyShimBlock = stickyEvents.length === 0 ? '' : `\ +;(function clayViteStickyEvents() { + var fired = {}; + var _orig = window.addEventListener.bind(window); + + window.addEventListener = function(type, handler, options) { + _orig(type, handler, options); + + if (Object.prototype.hasOwnProperty.call(fired, type)) { + Promise.resolve().then(function() { + handler(new CustomEvent(type, { detail: fired[type] })); + }); + } + }; + +${stickyListeners} +}()); +`; + + const globalsImport = (await fs.pathExists(GLOBALS_INIT_FILE)) + ? "import './_globals-init.js';\n" + : '// no global/js — skipping _globals-init\n'; + + const content = [ + '// AUTO-GENERATED — clay vite bootstrap (do not edit)', + `// ${new Date().toISOString()}`, + '// This file is the single ESM entry point injected into every page via', + '//