diff --git a/gnm/shape/demos/.gitignore b/gnm/shape/demos/.gitignore new file mode 100644 index 00000000..9c7c54c2 --- /dev/null +++ b/gnm/shape/demos/.gitignore @@ -0,0 +1,2 @@ +*.bin +*.mp4 diff --git a/gnm/shape/demos/web/GNMControls.js b/gnm/shape/demos/web/GNMControls.js new file mode 100644 index 00000000..2c648d49 --- /dev/null +++ b/gnm/shape/demos/web/GNMControls.js @@ -0,0 +1,751 @@ +/** + * GNMControls — DOM control panel for the GNM head demo. + * + * Provides the full parameter surface of the model: semantic sampling + * (gender/ethnicity identities, 20 expression classes), grouped PCA sliders + * for all 253 identity + 383 expression components, joint pose, animation + * drivers, and view options. Styling lives in gnm.css. + */ + +const MATERIAL_MODES = [ + ['studio', 'Studio'], + ['clay', 'Clay'], + ['normals', 'Normals'], + ['regions', 'Regions'], +]; + +const INITIAL_SLIDERS_PER_GROUP = 10; +const PARAM_RANGE = 3; + +function prettify(name) { + return name + .replace(/_region$/, '') + .replace(/_/g, ' ') + .replace(/^./, (c) => c.toUpperCase()); +} + +/** Splits ordered PCA names like head_000… into contiguous groups. */ +function groupParams(names) { + const groups = []; + let current = null; + names.forEach((name, index) => { + const key = name.replace(/_?\d+$/, '').replace(/_mean$/, ''); + if (!current || current.key !== key) { + current = {key, title: prettify(key), start: index, names: []}; + groups.push(current); + } + current.names.push(name); + }); + return groups; +} + +function gaussian() { + let u = 0; + while (u === 0) u = Math.random(); + return Math.sqrt(-2 * Math.log(u)) * Math.cos(2 * Math.PI * Math.random()); +} + +export class GNMControls { + constructor(model, samplers, scene) { + this.model = model; + this.samplers = samplers; + this.scene = scene; + this._paramInputs = new Map(); // 'identity:12' -> {input, output} + this._poseInputs = []; + scene.onModelChanged = () => this.syncParams(); + scene.onStatus = (text) => this.setStatus(text); + } + + attach() { + const meta = this.model.meta; + this.root = document.createElement('div'); + this.root.id = 'gnm-panel'; + // demo.css styles bare
as a fixed, full-width page banner, so the + // panel chrome uses scoped divs instead of semantic header/footer tags. + this.root.innerHTML = ` +
+
+

GNM Head Explorer

+

Generative aNthropometric Model v${meta.gnmVersion}

+
+ +
+
+ +
+
+ ${meta.numVertices.toLocaleString()} vertices · + ${(this.model.triangles.length / 3).toLocaleString()} triangles · + ${meta.identityDim + meta.expressionDim} parameters
+
+ + google/gnm — research model; see its README for representation + limitations.
+ google/xrblocks — the AI + XR SDK powering this + demo. +
`; + document.body.appendChild(this.root); + + this.statusElement = this.root.querySelector('#gnm-status'); + this.statsElement = this.root.querySelector('#gnm-stats'); + this.tabsElement = this.root.querySelector('#gnm-tabs'); + this.pagesElement = this.root.querySelector('#gnm-pages'); + this.root.querySelector('#gnm-collapse').addEventListener('click', () => { + this.root.classList.toggle('collapsed'); + this.root.querySelector('#gnm-collapse').textContent = + this.root.classList.contains('collapsed') ? '+' : '–'; + }); + + this._buildTabs([ + ['Sample', () => this._buildSampleTab()], + ['Identity', () => this._buildParamTab('identity')], + ['Expression', () => this._buildParamTab('expression')], + ['Pose', () => this._buildPoseTab()], + ['Animate', () => this._buildAnimateTab()], + ['View', () => this._buildViewTab()], + ]); + this._bindKeyboard(); + this._startStatsLoop(); + } + + // ------------------------------------------------------------------ tabs -- + + _buildTabs(tabs) { + this._pages = []; + tabs.forEach(([label, build], index) => { + const button = document.createElement('button'); + button.textContent = label; + button.addEventListener('click', () => this._selectTab(index)); + this.tabsElement.appendChild(button); + const page = document.createElement('div'); + page.className = 'gnm-page'; + this.pagesElement.appendChild(page); + const entry = {button, page, built: false}; + entry.buildInto = () => { + if (!entry.built) { + entry.built = true; + page.appendChild(build.call(this)); + } + }; + this._pages.push(entry); + }); + this._selectTab(0); + } + + _selectTab(index) { + this._pages.forEach((entry, i) => { + const active = i === index; + entry.button.classList.toggle('active', active); + entry.page.classList.toggle('active', active); + if (active) entry.buildInto(); + }); + this.syncParams(); + } + + _section(parent, title) { + const section = document.createElement('section'); + if (title) { + const heading = document.createElement('h2'); + heading.textContent = title; + section.appendChild(heading); + } + parent.appendChild(section); + return section; + } + + _button(parent, label, onClick, className = '') { + const button = document.createElement('button'); + button.className = `gnm-btn ${className}`; + button.textContent = label; + button.addEventListener('click', onClick); + parent.appendChild(button); + return button; + } + + _toggle(parent, label, initial, onChange) { + const row = document.createElement('label'); + row.className = 'gnm-toggle'; + const input = document.createElement('input'); + input.type = 'checkbox'; + input.checked = initial; + input.addEventListener('change', () => onChange(input.checked)); + row.appendChild(input); + row.appendChild(document.createTextNode(label)); + parent.appendChild(row); + return input; + } + + _slider(parent, label, min, max, step, value, onInput, onReset) { + const row = document.createElement('div'); + row.className = 'gnm-slider'; + const name = document.createElement('span'); + name.textContent = label; + const input = document.createElement('input'); + input.type = 'range'; + input.min = min; + input.max = max; + input.step = step; + input.value = value; + const output = document.createElement('output'); + output.textContent = (+value).toFixed(2); + input.addEventListener('input', () => { + output.textContent = (+input.value).toFixed(2); + onInput(+input.value); + }); + if (onReset) { + row.title = 'Double-click to reset'; + row.addEventListener('dblclick', () => { + input.value = onReset(); + output.textContent = (+input.value).toFixed(2); + }); + } + row.append(name, input, output); + parent.appendChild(row); + return {row, input, output}; + } + + // ---------------------------------------------------------------- sample -- + + _buildSampleTab() { + const fragment = document.createDocumentFragment(); + const identity = this._section(fragment, 'Semantic identity'); + identity.insertAdjacentHTML( + 'beforeend', + `
+ + +
` + ); + this._sigma = 0.9; + // Gender and ethnicity apply live (like the slider tabs); changing either + // re-rolls a new face of that demographic. + identity + .querySelector('#gnm-gender') + .addEventListener('change', () => this._sampleSemanticIdentity(true)); + identity + .querySelector('#gnm-ethnicity') + .addEventListener('change', () => this._sampleSemanticIdentity(true)); + + // The Variation slider is live too: it reuses the current face's random + // seed, so dragging exaggerates the same identity instead of re-rolling. + const sigmaRow = this._slider( + identity, + 'Variation', + 0, + 1.5, + 0.05, + 0.9, + (v) => { + this._sigma = v; + this._sampleSemanticIdentity(false, false); + } + ); + sigmaRow.row.classList.add('wide'); + const identityButtons = document.createElement('div'); + identityButtons.className = 'gnm-btn-row'; + this._button(identityButtons, 'New face', () => + this._sampleSemanticIdentity(true) + ); + this._button(identityButtons, 'Random mix', () => { + this.scene.sampleRandomIdentity(this._sigma); + this.setStatus('sampled a random identity mix'); + }); + this._button(identityButtons, 'Template', () => { + this.model.resetIdentity(); + this.syncParams(); + this.setStatus('template (mean) face'); + }); + identity.appendChild(identityButtons); + + const expression = this._section(fragment, 'Semantic expression'); + const chips = document.createElement('div'); + chips.className = 'gnm-chips'; + this.samplers.expressionClasses.forEach((label, index) => { + const chip = document.createElement('button'); + chip.className = 'gnm-chip'; + chip.textContent = prettify(label); + chip.addEventListener('click', () => { + this.scene.sampleExpression(index, this._sigma); + this.setStatus(`expression: ${prettify(label).toLowerCase()}`); + }); + chips.appendChild(chip); + }); + expression.appendChild(chips); + const expressionButtons = document.createElement('div'); + expressionButtons.className = 'gnm-btn-row'; + this._button(expressionButtons, 'Random blend', () => { + this.scene.sampleRandomExpression(this._sigma); + this.setStatus('random expression blend'); + }); + this._button(expressionButtons, 'Neutral', () => { + this.model.resetExpression(); + this.syncParams(); + this.setStatus('neutral expression'); + }); + expression.appendChild(expressionButtons); + return fragment; + } + + /** + * Samples an identity from the current gender/ethnicity selection. + * + * @param {boolean} reroll - When true, draws a fresh random seed (a new + * face). When false, reuses the last seed so only `sigma` changes, letting + * the Variation slider morph the same identity smoothly. + * @param {boolean} [smooth=true] - Whether to blend toward the new shape. + */ + _sampleSemanticIdentity(reroll, smooth = true) { + if (reroll || this._identitySeed === undefined) { + this._identitySeed = (Math.random() * 0x100000000) >>> 0; + } + const gender = this.root.querySelector('#gnm-gender').value; + const ethnicity = this.root.querySelector('#gnm-ethnicity').value; + const genderWeights = + gender === 'blend' + ? [1, 1] + : this.samplers.genders.map((g) => (g === gender ? 1 : 0)); + const ethnicityWeights = + ethnicity === 'blend' + ? [1, 1, 1, 1] + : this.samplers.ethnicities.map((e) => (e === ethnicity ? 1 : 0)); + // Reseed so the same selection + seed is reproducible; the Variation + // slider then only changes the latent scale, not the face. + this.samplers.seed(this._identitySeed); + this.scene.sampleIdentity( + genderWeights, + ethnicityWeights, + this._sigma, + smooth + ); + this.setStatus( + `sampled ${gender === 'blend' ? 'mixed' : gender} · ` + + `${ethnicity === 'blend' ? 'mixed' : prettify(ethnicity)}` + ); + } + + // ---------------------------------------------------------------- params -- + + _buildParamTab(kind) { + const fragment = document.createDocumentFragment(); + const names = + kind === 'identity' + ? this.model.meta.identityNames + : this.model.meta.expressionNames; + for (const group of groupParams(names)) { + const section = this._section( + fragment, + `${group.title} (${group.names.length})` + ); + const buttons = document.createElement('div'); + buttons.className = 'gnm-btn-row'; + this._button(buttons, 'Randomize', () => { + for (let i = 0; i < group.names.length; ++i) { + this._setParam(kind, group.start + i, gaussian()); + } + this.syncParams(); + }); + this._button(buttons, 'Zero', () => { + for (let i = 0; i < group.names.length; ++i) { + this._setParam(kind, group.start + i, 0); + } + this.syncParams(); + }); + section.appendChild(buttons); + const list = document.createElement('div'); + section.appendChild(list); + const buildRows = (from, to) => { + for (let i = from; i < to; ++i) { + const index = group.start + i; + const suffix = group.names[i].match(/(\d+|mean)$/)?.[1] ?? `${i}`; + const {input, output} = this._slider( + list, + suffix, + -PARAM_RANGE, + PARAM_RANGE, + 0.01, + this._getParam(kind, index), + (value) => { + this._setParam(kind, index, value); + this.scene.setPulse(kind, index); + }, + () => { + this._setParam(kind, index, 0); + return 0; + } + ); + this._paramInputs.set(`${kind}:${index}`, {input, output}); + } + }; + const initial = Math.min(INITIAL_SLIDERS_PER_GROUP, group.names.length); + buildRows(0, initial); + if (group.names.length > initial) { + const more = this._button( + section, + `Show all ${group.names.length}`, + () => { + buildRows(initial, group.names.length); + more.remove(); + this.syncParams(); + }, + 'ghost' + ); + } + } + return fragment; + } + + _getParam(kind, index) { + return kind === 'identity' + ? this.model.identity[index] + : this.model.expression[index]; + } + + _setParam(kind, index, value) { + if (kind === 'identity') this.model.setIdentityParam(index, value); + else this.model.setExpressionParam(index, value); + } + + // ------------------------------------------------------------------ pose -- + + _buildPoseTab() { + const fragment = document.createDocumentFragment(); + const tracking = this._section(fragment, 'Tracking'); + this._eyesToggle = this._toggle( + tracking, + 'Eyes follow camera', + this.scene.eyesFollowCamera, + (checked) => { + this.scene.eyesFollowCamera = checked; + this._updatePoseDisabled(); + } + ); + this._headToggle = this._toggle( + tracking, + 'Head follows camera', + this.scene.headFollowsCamera, + (checked) => { + this.scene.headFollowsCamera = checked; + this._updatePoseDisabled(); + } + ); + + const jointSpecs = [ + [0, 'Neck', 0.5], + [1, 'Head', 0.6], + [2, 'Left eye', 0.6], + [3, 'Right eye', 0.6], + ]; + const axes = ['pitch (x)', 'yaw (y)', 'roll (z)']; + for (const [joint, label, range] of jointSpecs) { + const section = this._section(fragment, label); + for (let axis = 0; axis < 3; ++axis) { + if (joint >= 2 && axis === 2) continue; // eye roll is not meaningful + const {input, output} = this._slider( + section, + axes[axis], + -range, + range, + 0.01, + this.model.rotations[joint * 3 + axis], + (value) => { + const o = joint * 3; + const r = this.model.rotations; + const next = [r[o], r[o + 1], r[o + 2]]; + next[axis] = value; + this.model.setJointRotation(joint, ...next); + this.scene._smoothedRotations.set(next, o); + } + ); + this._poseInputs.push({joint, axis, input, output}); + } + } + const translation = this._section(fragment, 'Translation (m)'); + ['x', 'y', 'z'].forEach((axis, index) => { + const {input, output} = this._slider( + translation, + axis, + -0.25, + 0.25, + 0.005, + this.model.translation[index], + (value) => { + const t = this.model.translation; + const next = [t[0], t[1], t[2]]; + next[index] = value; + this.model.setTranslation(...next); + } + ); + this._poseInputs.push({translationAxis: index, input, output}); + }); + const buttons = document.createElement('div'); + buttons.className = 'gnm-btn-row'; + this._button(buttons, 'Reset pose', () => { + this.model.resetPose(); + this.scene._smoothedRotations.fill(0); + this.syncParams(); + }); + fragment.appendChild(buttons); + this._updatePoseDisabled(); + return fragment; + } + + _updatePoseDisabled() { + for (const entry of this._poseInputs) { + if (entry.translationAxis !== undefined) continue; + const trackedEyes = this.scene.eyesFollowCamera && entry.joint >= 2; + const trackedHead = this.scene.headFollowsCamera && entry.joint === 1; + const sway = this.scene.idleSway && entry.joint === 0; + entry.input.disabled = trackedEyes || trackedHead || sway; + } + } + + // --------------------------------------------------------------- animate -- + + _buildAnimateTab() { + const fragment = document.createDocumentFragment(); + const drivers = this._section(fragment, 'Drivers'); + this._animToggles = { + tour: this._toggle( + drivers, + 'Expression tour (20 classes)', + false, + (checked) => this.scene.setExpressionTour(checked) + ), + morph: this._toggle( + drivers, + 'Identity morph (random walk)', + false, + (checked) => this.scene.setIdentityMorph(checked) + ), + pulse: this._toggle( + drivers, + 'Pulse last-touched slider ±2.3', + false, + (checked) => this.scene.setPulseEnabled(checked) + ), + sway: this._toggle(drivers, 'Idle neck sway', false, (checked) => { + this.scene.idleSway = checked; + this._updatePoseDisabled(); + }), + turntable: this._toggle(drivers, 'Turntable', false, (checked) => { + this.scene.turntable = checked; + }), + }; + const speed = this._section(fragment, 'Speed'); + this._slider(speed, 'Rate', 0.25, 2.5, 0.05, 1, (value) => { + this.scene.animationSpeed = value; + }); + const hint = document.createElement('p'); + hint.className = 'gnm-hint'; + hint.textContent = + 'Tip: touch any Identity/Expression slider first, then enable Pulse ' + + 'to see what that PCA component does.'; + fragment.appendChild(hint); + return fragment; + } + + // ------------------------------------------------------------------ view -- + + _buildViewTab() { + const fragment = document.createDocumentFragment(); + const material = this._section(fragment, 'Material'); + const modes = document.createElement('div'); + modes.className = 'gnm-btn-row'; + this._materialButtons = MATERIAL_MODES.map(([mode, label]) => { + const button = this._button( + modes, + label, + () => this.setMaterialMode(mode), + mode === this.scene.materialMode ? 'active' : '' + ); + button.dataset.mode = mode; + return button; + }); + material.appendChild(modes); + + const overlays = this._section(fragment, 'Overlays'); + this._wireToggle = this._toggle(overlays, 'Quad wireframe', false, (c) => + this.scene.setWireframeVisible(c) + ); + this._landmarkToggle = this._toggle( + overlays, + '68 facial landmarks', + false, + (c) => this.scene.setLandmarksVisible(c) + ); + this._skeletonToggle = this._toggle( + overlays, + 'Skeleton (neck → head → eyes)', + false, + (c) => this.scene.setSkeletonVisible(c) + ); + + const components = this._section(fragment, 'Mesh components'); + this._componentToggles = this.model.meta.componentNames.map((name, index) => + this._toggle(components, prettify(name), true, (checked) => + this.scene.setComponentVisible(index, checked) + ) + ); + + const exportSection = this._section(fragment, 'Export'); + const buttons = document.createElement('div'); + buttons.className = 'gnm-btn-row'; + this._button(buttons, 'Download OBJ', () => { + const blob = new Blob([this.scene.exportOBJ()], {type: 'text/plain'}); + const link = document.createElement('a'); + link.href = URL.createObjectURL(blob); + link.download = 'gnm_head.obj'; + link.click(); + URL.revokeObjectURL(link.href); + this.setStatus('exported gnm_head.obj'); + }); + exportSection.appendChild(buttons); + return fragment; + } + + setMaterialMode(mode) { + this.scene.setMaterialMode(mode); + this._materialButtons?.forEach((button) => + button.classList.toggle('active', button.dataset.mode === mode) + ); + } + + // ------------------------------------------------------------------ sync -- + + /** Reflects model parameter values into any built slider rows. */ + syncParams() { + for (const [key, {input, output}] of this._paramInputs) { + const [kind, index] = key.split(':'); + const value = this._getParam(kind, +index); + input.value = value; + output.textContent = value.toFixed(2); + } + for (const entry of this._poseInputs) { + const value = + entry.translationAxis !== undefined + ? this.model.translation[entry.translationAxis] + : this.model.rotations[entry.joint * 3 + entry.axis]; + entry.input.value = value; + entry.output.textContent = value.toFixed(2); + } + } + + setStatus(text) { + if (this.statusElement) this.statusElement.textContent = text; + } + + _startStatsLoop() { + let frames = 0; + let last = performance.now(); + const tick = () => { + frames++; + const now = performance.now(); + if (now - last >= 500) { + const fps = (frames * 1000) / (now - last); + this.statsElement.textContent = + `${fps.toFixed(0)} fps · mesh update ` + + `${this.scene.lastComputeMs.toFixed(1)} ms`; + frames = 0; + last = now; + // Keep sliders live while animation drivers run. + if ( + this.scene.tour || + this.scene.morph || + this.scene.pulseEnabled || + this.scene._idFade || + this.scene._exprFade + ) { + this.syncParams(); + } + // Reflect state changed from the spatial (XR) panels. + this._syncViewFromScene(); + } + requestAnimationFrame(tick); + }; + requestAnimationFrame(tick); + } + + /** Mirrors scene state into checkboxes/buttons built by this panel. */ + _syncViewFromScene() { + const scene = this.scene; + if (this._wireToggle) this._wireToggle.checked = scene.wireframe.visible; + if (this._landmarkToggle) { + this._landmarkToggle.checked = scene.landmarkMesh.visible; + } + if (this._skeletonToggle) { + this._skeletonToggle.checked = scene.skeletonGroup.visible; + } + this._componentToggles?.forEach((toggle, index) => { + toggle.checked = scene.visibleComponents[index]; + }); + this._materialButtons?.forEach((button) => + button.classList.toggle( + 'active', + button.dataset.mode === scene.materialMode + ) + ); + if (this._animToggles) { + this._animToggles.tour.checked = !!scene.tour; + this._animToggles.morph.checked = !!scene.morph; + this._animToggles.pulse.checked = scene.pulseEnabled; + this._animToggles.sway.checked = scene.idleSway; + this._animToggles.turntable.checked = scene.turntable; + } + if (this._eyesToggle && this._headToggle) { + const changed = + this._eyesToggle.checked !== scene.eyesFollowCamera || + this._headToggle.checked !== scene.headFollowsCamera; + this._eyesToggle.checked = scene.eyesFollowCamera; + this._headToggle.checked = scene.headFollowsCamera; + if (changed) this._updatePoseDisabled(); + } + } + + _bindKeyboard() { + document.addEventListener('keydown', (event) => { + const tag = document.activeElement?.tagName; + if (tag === 'INPUT' || tag === 'SELECT' || tag === 'TEXTAREA') return; + switch (event.key.toLowerCase()) { + case 'r': + this.scene.sampleRandomIdentity(this._sigma ?? 0.9); + this.setStatus('random identity (R)'); + break; + case 'e': + this.scene.sampleRandomExpression(this._sigma ?? 0.9); + this.setStatus('random expression (E)'); + break; + case 'n': + this.scene.resetToNeutral(); + this.syncParams(); + this.setStatus('neutral (N)'); + break; + case 't': + this.scene.setExpressionTour(!this.scene.tour); + break; + case 'w': + if (this._wireToggle) { + this._wireToggle.checked = !this._wireToggle.checked; + } + this.scene.setWireframeVisible( + this._wireToggle?.checked ?? !this.scene.wireframe.visible + ); + break; + case 'g': + this.scene.turntable = !this.scene.turntable; + break; + default: + return; + } + }); + } +} diff --git a/gnm/shape/demos/web/GNMModel.js b/gnm/shape/demos/web/GNMModel.js new file mode 100644 index 00000000..9034f1e8 --- /dev/null +++ b/gnm/shape/demos/web/GNMModel.js @@ -0,0 +1,551 @@ +/** + * GNMModel.js — JavaScript port of the GNM (Generative aNthropometric Model) + * head forward function, mirroring gnm/shape/gnm_common.py: + * + * bind = template + identity_basis^T id + expression_basis^T expr + * joints = template_joints + joint_identity_basis^T id + * world = LinearBlendSkinning(bind + pose_correctives, FK(joints, rotations)) + * + * The model data is read from a 'GNMW' container produced by + * tools/export_gnm_web.py. The large PCA bases are int8-quantized with one + * float32 scale per component; coefficients fold the scale in at accumulation + * time so dequantization is free. + * + * This file is dependency-free (typed arrays only) so it runs in Node for + * verification as well as in the browser. + */ + +const MAGIC = 0x474e4d57; // 'GNMW' +const EPSILON = 1e-8; +// Full re-accumulation after this many incremental slider updates bounds +// float32 drift. +const MAX_INCREMENTAL_UPDATES = 2000; + +const DTYPE_CTORS = { + float32: Float32Array, + int8: Int8Array, + uint8: Uint8Array, + uint16: Uint16Array, + int32: Int32Array, +}; + +/** Parses a GNMW container buffer into {meta, sections}. */ +export function parseContainer(buffer) { + const view = new DataView(buffer); + const magic = + (view.getUint8(0) << 24) | + (view.getUint8(1) << 16) | + (view.getUint8(2) << 8) | + view.getUint8(3); + if (magic !== MAGIC) { + throw new Error('Not a GNMW container.'); + } + const version = view.getUint32(4, true); + if (version !== 1) { + throw new Error(`Unsupported GNMW version ${version}.`); + } + const headerLength = view.getUint32(8, true); + const headerText = new TextDecoder().decode( + new Uint8Array(buffer, 12, headerLength) + ); + const header = JSON.parse(headerText); + const base = 12 + headerLength; + const sections = {}; + for (const section of header.sections) { + const Ctor = DTYPE_CTORS[section.dtype]; + sections[section.name] = new Ctor( + buffer, + base + section.offset, + section.byteLength / Ctor.BYTES_PER_ELEMENT + ); + } + return {meta: header.meta, sections}; +} + +/** Fetches a URL into an ArrayBuffer, reporting streaming progress. */ +export async function fetchWithProgress(url, onProgress) { + const response = await fetch(url); + if (!response.ok) { + throw new Error(`Failed to fetch ${url}: ${response.status}`); + } + const total = Number(response.headers.get('Content-Length')) || 0; + if (!response.body || !total) { + return await response.arrayBuffer(); + } + const reader = response.body.getReader(); + const data = new Uint8Array(total); + let received = 0; + for (;;) { + const {done, value} = await reader.read(); + if (done) break; + if (received + value.length > total) { + // Content-Length lied (e.g. proxy); fall back to chunk list. + const chunks = [data.subarray(0, received), value]; + for (;;) { + const next = await reader.read(); + if (next.done) break; + chunks.push(next.value); + } + const length = chunks.reduce((n, c) => n + c.length, 0); + const out = new Uint8Array(length); + let offset = 0; + for (const chunk of chunks) { + out.set(chunk, offset); + offset += chunk.length; + } + return out.buffer; + } + data.set(value, received); + received += value.length; + if (onProgress) onProgress(received / total); + } + return data.buffer.byteLength === received + ? data.buffer + : data.slice(0, received).buffer; +} + +/** Rodrigues' formula matching gnm_common.axis_angle_to_rotation_matrix. */ +export function axisAngleToMatrix(x, y, z, out, offset = 0) { + const normSquared = x * x + y * y + z * z; + const angle = Math.sqrt(Math.max(normSquared, EPSILON)); + const ax = x / angle; + const ay = y / angle; + const az = z / angle; + const s = Math.sin(angle); + const c = Math.cos(angle); + const t = 1 - c; + out[offset + 0] = c + t * ax * ax; + out[offset + 1] = t * ax * ay - s * az; + out[offset + 2] = t * ax * az + s * ay; + out[offset + 3] = t * ax * ay + s * az; + out[offset + 4] = c + t * ay * ay; + out[offset + 5] = t * ay * az - s * ax; + out[offset + 6] = t * ax * az - s * ay; + out[offset + 7] = t * ay * az + s * ax; + out[offset + 8] = c + t * az * az; +} + +export class GNMHeadModel { + constructor(meta, sections) { + this.meta = meta; + this.numVertices = meta.numVertices; + this.numJoints = meta.numJoints; + this.identityDim = meta.identityDim; + this.expressionDim = meta.expressionDim; + + this.template = sections.template; + this.triangles = sections.triangles; + this.quads = sections.quads; + this.templateJoints = sections.template_joints; + this.jointParents = sections.joint_parents; + this.skinningWeights = sections.skinning_weights; + this.jointIdentityBasis = sections.joint_identity_basis; + this.identityBasis = sections.identity_basis; + this.identityScales = sections.identity_scales; + this.expressionBasis = sections.expression_basis; + this.expressionScales = sections.expression_scales; + this.componentId = sections.component_id; + this.materialId = sections.material_id; + this.regionId = sections.region_id; + this.landmarkIndices = sections.landmark_indices; + this.landmarkWeights = sections.landmark_weights; + + const v3 = this.numVertices * 3; + // Parameters. + this.identity = new Float32Array(this.identityDim); + this.expression = new Float32Array(this.expressionDim); + this.rotations = new Float32Array(this.numJoints * 3); + this.translation = new Float32Array(3); + + // Cached linear-basis sums (bind-pose displacements). + this._idSum = new Float32Array(v3); + this._exprSum = new Float32Array(v3); + this._incrementalUpdates = 0; + + // Blend states (null when inactive). + this._idBlend = null; + this._exprBlend = null; + + // Scratch buffers for the pose pipeline. + this._bind = new Float32Array(v3); + this._jointsBind = new Float32Array(this.numJoints * 3); + this._rotWorld = new Float32Array(this.numJoints * 9); + this._skinTrans = new Float32Array(this.numJoints * 3); + this.jointsWorld = new Float32Array(this.numJoints * 3); + + this.dirty = true; + } + + static async load(url, onProgress) { + const buffer = await fetchWithProgress(url, onProgress); + const {meta, sections} = parseContainer(buffer); + return new GNMHeadModel(meta, sections); + } + + // ---------------------------------------------------------------- params -- + + /** Adds `factor` times basis component `index` into `sum`. */ + _addComponent(sum, basis, index, factor) { + if (factor === 0) return; + const offset = index * sum.length; + for (let j = 0, n = sum.length; j < n; ++j) { + sum[j] += basis[offset + j] * factor; + } + } + + _accumulate(sum, basis, scales, coefficients) { + sum.fill(0); + for (let i = 0; i < coefficients.length; ++i) { + const c = coefficients[i]; + if (c !== 0) this._addComponent(sum, basis, i, c * scales[i]); + } + } + + _maybeReaccumulate() { + if (++this._incrementalUpdates < MAX_INCREMENTAL_UPDATES) return; + this._incrementalUpdates = 0; + this._accumulate( + this._idSum, + this.identityBasis, + this.identityScales, + this.identity + ); + this._accumulate( + this._exprSum, + this.expressionBasis, + this.expressionScales, + this.expression + ); + } + + setIdentityParam(index, value) { + const delta = value - this.identity[index]; + if (delta === 0) return; + this.identity[index] = value; + this._addComponent( + this._idSum, + this.identityBasis, + index, + delta * this.identityScales[index] + ); + this._maybeReaccumulate(); + this.dirty = true; + } + + setExpressionParam(index, value) { + const delta = value - this.expression[index]; + if (delta === 0) return; + this.expression[index] = value; + this._addComponent( + this._exprSum, + this.expressionBasis, + index, + delta * this.expressionScales[index] + ); + this._maybeReaccumulate(); + this.dirty = true; + } + + setIdentityVector(values) { + this.identity.set(values.subarray ? values : Float32Array.from(values)); + this._accumulate( + this._idSum, + this.identityBasis, + this.identityScales, + this.identity + ); + this._idBlend = null; + this.dirty = true; + } + + setExpressionVector(values) { + this.expression.set(values.subarray ? values : Float32Array.from(values)); + this._accumulate( + this._exprSum, + this.expressionBasis, + this.expressionScales, + this.expression + ); + this._exprBlend = null; + this.dirty = true; + } + + resetIdentity() { + this.setIdentityVector(new Float32Array(this.identityDim)); + } + + resetExpression() { + this.setExpressionVector(new Float32Array(this.expressionDim)); + } + + setJointRotation(jointIndex, x, y, z) { + const o = jointIndex * 3; + this.rotations[o] = x; + this.rotations[o + 1] = y; + this.rotations[o + 2] = z; + this.dirty = true; + } + + setTranslation(x, y, z) { + this.translation[0] = x; + this.translation[1] = y; + this.translation[2] = z; + this.dirty = true; + } + + resetPose() { + this.rotations.fill(0); + this.translation.fill(0); + this.dirty = true; + } + + // ---------------------------------------------------------------- blends -- + // Blending exploits linearity: sum(lerp(a, b, t)) == lerp(sumA, sumB, t), + // so a full-vector crossfade costs one lerp over V*3 floats per frame + // instead of a (dim × V × 3) re-accumulation. + + _beginBlend(kind, target, basis, scales, current, currentSum) { + const targetArray = Float32Array.from(target); + const sumB = new Float32Array(currentSum.length); + this._accumulate(sumB, basis, scales, targetArray); + this[kind] = { + coeffA: Float32Array.from(current), + coeffB: targetArray, + sumA: Float32Array.from(currentSum), + sumB, + }; + } + + beginIdentityBlend(target) { + this._beginBlend( + '_idBlend', + target, + this.identityBasis, + this.identityScales, + this.identity, + this._idSum + ); + } + + beginExpressionBlend(target) { + this._beginBlend( + '_exprBlend', + target, + this.expressionBasis, + this.expressionScales, + this.expression, + this._exprSum + ); + } + + _applyBlend(blend, coefficients, sum, t) { + const s = 1 - t; + const {coeffA, coeffB, sumA, sumB} = blend; + for (let i = 0; i < coefficients.length; ++i) { + coefficients[i] = coeffA[i] * s + coeffB[i] * t; + } + for (let j = 0, n = sum.length; j < n; ++j) { + sum[j] = sumA[j] * s + sumB[j] * t; + } + this.dirty = true; + } + + setIdentityBlend(t) { + if (this._idBlend) { + this._applyBlend(this._idBlend, this.identity, this._idSum, t); + } + } + + setExpressionBlend(t) { + if (this._exprBlend) { + this._applyBlend(this._exprBlend, this.expression, this._exprSum, t); + } + } + + // --------------------------------------------------------------- forward -- + + _computeJointsBind() { + const out = this._jointsBind; + out.set(this.templateJoints); + const basis = this.jointIdentityBasis; + const stride = this.numJoints * 3; + for (let i = 0; i < this.identityDim; ++i) { + const c = this.identity[i]; + if (c === 0) continue; + const offset = i * stride; + for (let k = 0; k < stride; ++k) { + out[k] += basis[offset + k] * c; + } + } + } + + /** + * Forward kinematics matching gnm_common.joint_transforms_world, followed + * by the skinning-transform construction from linear_blend_skinning: + * per joint, rotation R_world and translation t_world − R_world·j_bind. + */ + _computeJointTransforms() { + const J = this.numJoints; + const joints = this._jointsBind; + const parents = this.jointParents; + const rotWorld = this._rotWorld; + const localRot = new Float32Array(9); + const worldTrans = this.jointsWorld; + + for (let j = 0; j < J; ++j) { + axisAngleToMatrix( + this.rotations[j * 3], + this.rotations[j * 3 + 1], + this.rotations[j * 3 + 2], + localRot, + 0 + ); + let lx, ly, lz; + if (j === 0) { + lx = joints[0] + this.translation[0]; + ly = joints[1] + this.translation[1]; + lz = joints[2] + this.translation[2]; + rotWorld.set(localRot, 0); + worldTrans[0] = lx; + worldTrans[1] = ly; + worldTrans[2] = lz; + } else { + const p = parents[j]; + lx = joints[j * 3] - joints[p * 3]; + ly = joints[j * 3 + 1] - joints[p * 3 + 1]; + lz = joints[j * 3 + 2] - joints[p * 3 + 2]; + const po = p * 9; + const jo = j * 9; + // rotWorld[j] = rotWorld[p] * localRot + for (let r = 0; r < 3; ++r) { + for (let c = 0; c < 3; ++c) { + rotWorld[jo + r * 3 + c] = + rotWorld[po + r * 3] * localRot[c] + + rotWorld[po + r * 3 + 1] * localRot[3 + c] + + rotWorld[po + r * 3 + 2] * localRot[6 + c]; + } + } + // worldTrans[j] = rotWorld[p] * local + worldTrans[p] + worldTrans[j * 3] = + rotWorld[po] * lx + + rotWorld[po + 1] * ly + + rotWorld[po + 2] * lz + + worldTrans[p * 3]; + worldTrans[j * 3 + 1] = + rotWorld[po + 3] * lx + + rotWorld[po + 4] * ly + + rotWorld[po + 5] * lz + + worldTrans[p * 3 + 1]; + worldTrans[j * 3 + 2] = + rotWorld[po + 6] * lx + + rotWorld[po + 7] * ly + + rotWorld[po + 8] * lz + + worldTrans[p * 3 + 2]; + } + } + + // Skinning translation: t_world − R_world · j_bind. + const skinTrans = this._skinTrans; + for (let j = 0; j < J; ++j) { + const jo = j * 9; + const bx = joints[j * 3]; + const by = joints[j * 3 + 1]; + const bz = joints[j * 3 + 2]; + skinTrans[j * 3] = + worldTrans[j * 3] - + (rotWorld[jo] * bx + rotWorld[jo + 1] * by + rotWorld[jo + 2] * bz); + skinTrans[j * 3 + 1] = + worldTrans[j * 3 + 1] - + (rotWorld[jo + 3] * bx + rotWorld[jo + 4] * by + rotWorld[jo + 5] * bz); + skinTrans[j * 3 + 2] = + worldTrans[j * 3 + 2] - + (rotWorld[jo + 6] * bx + rotWorld[jo + 7] * by + rotWorld[jo + 8] * bz); + } + } + + /** + * Runs the full forward pass and writes world-space vertices into `out` + * (Float32Array of length numVertices*3). Also refreshes `jointsWorld`. + */ + computeVertices(out) { + const V = this.numVertices; + const bind = this._bind; + const template = this.template; + const idSum = this._idSum; + const exprSum = this._exprSum; + for (let j = 0, n = V * 3; j < n; ++j) { + bind[j] = template[j] + idSum[j] + exprSum[j]; + } + + this._computeJointsBind(); + this._computeJointTransforms(); + + const weights = this.skinningWeights; + const rotWorld = this._rotWorld; + const skinTrans = this._skinTrans; + const J = this.numJoints; + for (let v = 0; v < V; ++v) { + const px = bind[v * 3]; + const py = bind[v * 3 + 1]; + const pz = bind[v * 3 + 2]; + let ox = 0; + let oy = 0; + let oz = 0; + for (let j = 0; j < J; ++j) { + const w = weights[j * V + v]; + if (w === 0) continue; + const jo = j * 9; + ox += + w * + (rotWorld[jo] * px + + rotWorld[jo + 1] * py + + rotWorld[jo + 2] * pz + + skinTrans[j * 3]); + oy += + w * + (rotWorld[jo + 3] * px + + rotWorld[jo + 4] * py + + rotWorld[jo + 5] * pz + + skinTrans[j * 3 + 1]); + oz += + w * + (rotWorld[jo + 6] * px + + rotWorld[jo + 7] * py + + rotWorld[jo + 8] * pz + + skinTrans[j * 3 + 2]); + } + out[v * 3] = ox; + out[v * 3 + 1] = oy; + out[v * 3 + 2] = oz; + } + this.dirty = false; + } + + /** Barycentric landmark extraction (68 × 3 vertices/weights). */ + computeLandmarks(vertices, out) { + const indices = this.landmarkIndices; + const weights = this.landmarkWeights; + const count = indices.length / 3; + for (let l = 0; l < count; ++l) { + let x = 0; + let y = 0; + let z = 0; + for (let k = 0; k < 3; ++k) { + const vi = indices[l * 3 + k] * 3; + const w = weights[l * 3 + k]; + x += vertices[vi] * w; + y += vertices[vi + 1] * w; + z += vertices[vi + 2] * w; + } + out[l * 3] = x; + out[l * 3 + 1] = y; + out[l * 3 + 2] = z; + } + return count; + } + + /** World-space rotation frame (row-major 3x3) of a joint after FK. */ + getJointWorldRotation(jointIndex) { + return this._rotWorld.subarray(jointIndex * 9, jointIndex * 9 + 9); + } +} diff --git a/gnm/shape/demos/web/GNMScene.js b/gnm/shape/demos/web/GNMScene.js new file mode 100644 index 00000000..de734997 --- /dev/null +++ b/gnm/shape/demos/web/GNMScene.js @@ -0,0 +1,708 @@ +import * as THREE from 'three'; +import {raycastSortFunction} from 'uiblocks'; +import * as xb from 'xrblocks'; + +import {GNMSpatialUI} from './GNMSpatialUI.js'; + +/** + * GNMScene — renders the GNM parametric head and animates it. + * + * Owns the Three.js mesh (positions streamed from GNMHeadModel), the material + * modes, wireframe / landmark / skeleton overlays, per-component visibility, + * gaze tracking, and the animation drivers (expression tour, identity morph, + * component pulse, idle sway, turntable). + */ + +const WORLD_HEAD_Y = 1.35; // where to place the head center in world space +const WORLD_HEAD_Z = -0.62; + +// Per-vertex material palette (indexed by material_id from the exporter: +// skin, teeth, gums, tongue, scleras, irises, pupils). +const MATERIAL_COLORS = [ + '#d9a183', + '#f2eddc', + '#c4726b', + '#b44f48', + '#f4f2ec', + '#5d4630', + '#100e0d', +]; + +const JOINT_LABELS = ['neck', 'head', 'left eye', 'right eye']; + +const TOUR_FADE_SECONDS = 0.9; +const TOUR_HOLD_SECONDS = 1.3; +const MORPH_FADE_SECONDS = 1.6; +const MORPH_HOLD_SECONDS = 0.9; + +export class GNMScene extends xb.Script { + constructor(model, samplers) { + super(); + this.model = model; + this.samplers = samplers; + + // View state. + this.materialMode = 'studio'; + this.visibleComponents = new Array(model.meta.componentNames.length).fill( + true + ); + + // Tracking / animation state. + this.eyesFollowCamera = true; + this.headFollowsCamera = false; + this.idleSway = false; + this.turntable = false; + this.animationSpeed = 1; + this.tour = null; // {phase, t, classIndex} + this.morph = null; // {phase, t} + this.pulse = null; // {kind: 'identity'|'expression', index, base} + this.pulseEnabled = false; + this._pulseTime = 0; + this._time = 0; + this._smoothedRotations = new Float32Array(model.numJoints * 3); + + this.lastComputeMs = 0; + /** Called when identity/expression change from inside the scene. */ + this.onModelChanged = null; + /** Called with a status string (e.g. current tour expression). */ + this.onStatus = null; + + // uiblocks UICore must be constructed with the owning Script. + this.spatialUI = new GNMSpatialUI(this); + } + + init() { + const model = this.model; + + // The head + overlays live under `anchor`, which is wrapped in a + // ModelViewer so the user can grab the pedestal to move it and grab the + // head to rotate it. The ModelViewer is positioned in _setupModelViewer() + // once the mesh (and thus its bounding box) exists. + this.anchor = new THREE.Group(); + this.modelViewer = new xb.ModelViewer({}); + this.modelViewer.add(this.anchor); + this.add(this.modelViewer); + + // ---- Geometry -------------------------------------------------------- + const geometry = new THREE.BufferGeometry(); + this.positionAttribute = new THREE.BufferAttribute( + new Float32Array(model.numVertices * 3), + 3 + ); + this.positionAttribute.setUsage(THREE.DynamicDrawUsage); + geometry.setAttribute('position', this.positionAttribute); + geometry.setAttribute( + 'color', + new THREE.BufferAttribute(this._buildMaterialColors(), 3) + ); + this._regionColors = null; // built lazily for the regions mode + this.fullIndex = new THREE.BufferAttribute(model.triangles, 1); + geometry.setIndex(this.fullIndex); + this.geometry = geometry; + + this.materials = { + studio: new THREE.MeshStandardMaterial({ + vertexColors: true, + roughness: 0.58, + metalness: 0.02, + polygonOffset: true, + polygonOffsetFactor: 1, + polygonOffsetUnits: 1, + }), + clay: new THREE.MeshStandardMaterial({ + color: 0xb9b2aa, + roughness: 0.82, + metalness: 0.0, + polygonOffset: true, + polygonOffsetFactor: 1, + polygonOffsetUnits: 1, + }), + normals: new THREE.MeshNormalMaterial({ + polygonOffset: true, + polygonOffsetFactor: 1, + polygonOffsetUnits: 1, + }), + regions: new THREE.MeshStandardMaterial({ + vertexColors: true, + roughness: 0.75, + metalness: 0.0, + polygonOffset: true, + polygonOffsetFactor: 1, + polygonOffsetUnits: 1, + }), + }; + this.mesh = new THREE.Mesh(geometry, this.materials.studio); + this.mesh.frustumCulled = false; + this.mesh.name = 'GNM Head'; + this.anchor.add(this.mesh); + + // ---- Wireframe (quad edges, sharing the position attribute) ---------- + const wireGeometry = new THREE.BufferGeometry(); + wireGeometry.setAttribute('position', this.positionAttribute); + wireGeometry.setIndex( + new THREE.BufferAttribute(this._buildQuadEdgeIndex(), 1) + ); + this.wireframe = new THREE.LineSegments( + wireGeometry, + new THREE.LineBasicMaterial({ + color: 0x30343c, + transparent: true, + opacity: 0.45, + }) + ); + this.wireframe.frustumCulled = false; + this.wireframe.visible = false; + // Overlay: never a reticle target — lines return no surface normal, which + // would otherwise trap the reticle at the origin (it also occludes the head + // because THREE raycasts objects even while hidden). + this.wireframe.ignoreReticleRaycast = true; + this.anchor.add(this.wireframe); + + // ---- Landmarks ------------------------------------------------------- + const landmarkCount = model.landmarkIndices.length / 3; + this.landmarkPoints = new Float32Array(landmarkCount * 3); + this.landmarkMesh = new THREE.InstancedMesh( + new THREE.SphereGeometry(0.0021, 10, 8), + new THREE.MeshBasicMaterial({color: 0x2fe3a8}), + landmarkCount + ); + this.landmarkMesh.frustumCulled = false; + this.landmarkMesh.visible = false; + this.landmarkMesh.ignoreReticleRaycast = true; // overlay, not a target + this.anchor.add(this.landmarkMesh); + + // ---- Skeleton -------------------------------------------------------- + this.jointMesh = new THREE.InstancedMesh( + new THREE.SphereGeometry(0.0055, 12, 10), + new THREE.MeshBasicMaterial({color: 0xffb037, depthTest: false}), + model.numJoints + ); + this.jointMesh.renderOrder = 10; + this.jointMesh.frustumCulled = false; + const bonePositions = new Float32Array((model.numJoints - 1) * 2 * 3); + this.boneAttribute = new THREE.BufferAttribute(bonePositions, 3); + this.boneAttribute.setUsage(THREE.DynamicDrawUsage); + const boneGeometry = new THREE.BufferGeometry(); + boneGeometry.setAttribute('position', this.boneAttribute); + this.boneLines = new THREE.LineSegments( + boneGeometry, + new THREE.LineBasicMaterial({color: 0xffb037, depthTest: false}) + ); + this.boneLines.renderOrder = 10; + this.boneLines.frustumCulled = false; + this.skeletonGroup = new THREE.Group(); + this.skeletonGroup.add(this.jointMesh, this.boneLines); + this.skeletonGroup.visible = false; + this.skeletonGroup.ignoreReticleRaycast = true; // overlay, not a target + this.anchor.add(this.skeletonGroup); + + // ---- Lights ---------------------------------------------------------- + const key = new THREE.DirectionalLight(0xfff1e0, 2.6); + key.position.set(0.7, 2.3, 0.9); + const fill = new THREE.DirectionalLight(0xbdd2ff, 1.0); + fill.position.set(-1.1, 1.4, 0.4); + const rim = new THREE.DirectionalLight(0xffffff, 1.6); + rim.position.set(-0.2, 1.9, -1.4); + const hemisphere = new THREE.HemisphereLight(0x93a4c0, 0x40382f, 0.9); + for (const light of [key, fill, rim]) { + light.target.position.set(0, WORLD_HEAD_Y, WORLD_HEAD_Z); + this.add(light.target); + this.add(light); + } + this.add(hemisphere); + + // Required for raycasting against uiblocks panels. + if (xb.core.input?.raycaster) { + xb.core.input.raycaster.sortFunction = raycastSortFunction; + } + this.spatialUI.build(); + + // First shape. + this.model.dirty = true; + this._refreshGeometry(); + + // Now that geometry exists, wrap it in the ModelViewer pedestal. + this._setupModelViewer(); + } + + /** + * Configures the ModelViewer around the head: a rotation proxy (grab the + * head to rotate) and a platform (grab the pedestal to move it), then places + * the whole thing so the head centre sits at the intended world position. + */ + _setupModelViewer() { + const viewer = this.modelViewer; + viewer.setupBoundingBox(); + viewer.setupRaycastCylinder(); + // A tighter pedestal than the default so it reads as a stand for the bust. + viewer.setupPlatform(new THREE.Vector2(0.12, 0.12)); + const size = new THREE.Vector3(); + viewer.bbox.getSize(size); + viewer.position.set(0, WORLD_HEAD_Y - size.y / 2, WORLD_HEAD_Z); + this._modelViewerHomeY = viewer.position.y; + } + + // -------------------------------------------------------------- helpers -- + + _buildMaterialColors() { + const model = this.model; + const palette = MATERIAL_COLORS.map((c) => new THREE.Color(c)); + const colors = new Float32Array(model.numVertices * 3); + for (let v = 0; v < model.numVertices; ++v) { + const color = palette[model.materialId[v]] ?? palette[0]; + colors[v * 3] = color.r; + colors[v * 3 + 1] = color.g; + colors[v * 3 + 2] = color.b; + } + return colors; + } + + _buildRegionColors() { + const model = this.model; + const regionCount = model.meta.regionNames.length; + const palette = []; + for (let i = 0; i < regionCount; ++i) { + palette.push(new THREE.Color().setHSL((i * 0.61803) % 1, 0.62, 0.55)); + } + const neutral = new THREE.Color('#6d6d70'); + const colors = new Float32Array(model.numVertices * 3); + for (let v = 0; v < model.numVertices; ++v) { + const id = model.regionId[v]; + const color = id === 255 ? neutral : palette[id]; + colors[v * 3] = color.r; + colors[v * 3 + 1] = color.g; + colors[v * 3 + 2] = color.b; + } + return colors; + } + + _buildQuadEdgeIndex() { + const quads = this.model.quads; + const V = this.model.numVertices; + const seen = new Set(); + const edges = []; + for (let q = 0; q < quads.length; q += 4) { + for (let k = 0; k < 4; ++k) { + const a = quads[q + k]; + const b = quads[q + ((k + 1) % 4)]; + const key = a < b ? a * V + b : b * V + a; + if (seen.has(key)) continue; + seen.add(key); + edges.push(a, b); + } + } + return Uint16Array.from(edges); + } + + /** Sends a status line to both the DOM panel and the spatial panels. */ + _emitStatus(text) { + this.onStatus?.(text); + this.spatialUI?.setStatus(text); + } + + // ------------------------------------------------------------ view state -- + + setMaterialMode(mode) { + if (!this.materials[mode]) return; + this.materialMode = mode; + if (mode === 'regions' && !this._regionColors) { + this._regionColors = new THREE.BufferAttribute( + this._buildRegionColors(), + 3 + ); + } + if (mode === 'regions') { + this.geometry.setAttribute('color', this._regionColors); + } else if (mode === 'studio') { + this.geometry.setAttribute( + 'color', + new THREE.BufferAttribute(this._buildMaterialColors(), 3) + ); + } + this.mesh.material = this.materials[mode]; + } + + setWireframeVisible(visible) { + this.wireframe.visible = visible; + } + + setLandmarksVisible(visible) { + this.landmarkMesh.visible = visible; + if (visible) this._updateLandmarks(); + } + + setSkeletonVisible(visible) { + this.skeletonGroup.visible = visible; + if (visible) this._updateSkeleton(); + } + + setComponentVisible(componentIndex, visible) { + this.visibleComponents[componentIndex] = visible; + const allVisible = this.visibleComponents.every(Boolean); + if (allVisible) { + this.geometry.setIndex(this.fullIndex); + return; + } + const {triangles, componentId} = this.model; + const filtered = []; + for (let t = 0; t < triangles.length; t += 3) { + if (this.visibleComponents[componentId[triangles[t]]]) { + filtered.push(triangles[t], triangles[t + 1], triangles[t + 2]); + } + } + this.geometry.setIndex( + new THREE.BufferAttribute(Uint16Array.from(filtered), 1) + ); + } + + // ------------------------------------------------------------- sampling -- + + sampleIdentity(genderWeights, ethnicityWeights, sigma, smooth = true) { + const target = this.samplers.sampleIdentity( + genderWeights, + ethnicityWeights, + sigma + ); + this._applyIdentity(target, smooth); + return target; + } + + sampleRandomIdentity(sigma = 1) { + this._applyIdentity(this.samplers.randomIdentity(sigma), true); + } + + sampleExpression(classWeights, sigma, smooth = true) { + const target = this.samplers.sampleExpression(classWeights, sigma); + this._applyExpression(target, smooth); + return target; + } + + sampleRandomExpression(sigma = 1) { + this._applyExpression(this.samplers.randomExpression(sigma), true); + } + + resetToNeutral() { + this.setExpressionTour(false); + this.setIdentityMorph(false); + this.model.resetExpression(); + this.model.resetPose(); + this._smoothedRotations.fill(0); + this.onModelChanged?.(); + } + + _applyIdentity(target, smooth) { + this.morph = null; + if (smooth) { + this.model.beginIdentityBlend(target); + this._idFade = {t: 0}; + } else { + this.model.setIdentityVector(target); + } + this.onModelChanged?.(); + } + + _applyExpression(target, smooth) { + this.tour = null; + if (smooth) { + this.model.beginExpressionBlend(target); + this._exprFade = {t: 0}; + } else { + this.model.setExpressionVector(target); + } + this.onModelChanged?.(); + } + + // ------------------------------------------------------------ animation -- + + setExpressionTour(enabled) { + this.tour = enabled ? {phase: 'fade', t: 0, classIndex: 0} : null; + if (enabled) { + this._startTourStep(0); + } else { + this._emitStatus(''); + } + } + + setIdentityMorph(enabled) { + this.morph = enabled ? {phase: 'fade', t: 0} : null; + if (enabled) { + this.model.beginIdentityBlend(this.samplers.randomIdentity(1)); + } + } + + setPulse(kind, index) { + this.pulse = {kind, index, base: this._paramValue(kind, index)}; + this._pulseTime = 0; + } + + setPulseEnabled(enabled) { + if (!enabled && this.pulse) { + this._setParamValue(this.pulse.kind, this.pulse.index, this.pulse.base); + this.onModelChanged?.(); + } + this.pulseEnabled = enabled; + this._pulseTime = 0; + if (enabled && this.pulse) { + this.pulse.base = this._paramValue(this.pulse.kind, this.pulse.index); + } + } + + _paramValue(kind, index) { + return kind === 'identity' + ? this.model.identity[index] + : this.model.expression[index]; + } + + _setParamValue(kind, index, value) { + if (kind === 'identity') this.model.setIdentityParam(index, value); + else this.model.setExpressionParam(index, value); + } + + _startTourStep(classIndex) { + const target = this.samplers.sampleExpression(classIndex, 0.85); + this.model.beginExpressionBlend(target); + this.tour = {phase: 'fade', t: 0, classIndex}; + const label = this.samplers.expressionClasses[classIndex].replace( + /_/g, + ' ' + ); + this._emitStatus(`expression tour — ${label}`); + } + + _updateTour(dt) { + const tour = this.tour; + tour.t += dt; + if (tour.phase === 'fade') { + const t = Math.min(tour.t / TOUR_FADE_SECONDS, 1); + this.model.setExpressionBlend(smoothstep(t)); + if (t >= 1) { + tour.phase = 'hold'; + tour.t = 0; + } + } else if (tour.t >= TOUR_HOLD_SECONDS) { + const next = + (tour.classIndex + 1) % this.samplers.expressionClasses.length; + this._startTourStep(next); + } + } + + _updateMorph(dt) { + const morph = this.morph; + morph.t += dt; + if (morph.phase === 'fade') { + const t = Math.min(morph.t / MORPH_FADE_SECONDS, 1); + this.model.setIdentityBlend(smoothstep(t)); + if (t >= 1) { + morph.phase = 'hold'; + morph.t = 0; + } + } else if (morph.t >= MORPH_HOLD_SECONDS) { + this.model.beginIdentityBlend(this.samplers.randomIdentity(1)); + morph.phase = 'fade'; + morph.t = 0; + } + } + + _updatePulse(dt) { + if (!this.pulseEnabled || !this.pulse) return; + this._pulseTime += dt; + const value = this.pulse.base + 2.3 * Math.sin(this._pulseTime * 2.6); + this._setParamValue(this.pulse.kind, this.pulse.index, value); + } + + _updateGaze(dt) { + const camera = xb.core?.camera; + if (!camera) return; + const model = this.model; + const target = this._tmpTarget ?? (this._tmpTarget = new THREE.Vector3()); + camera.getWorldPosition(target); + this.anchor.worldToLocal(target); + + const setLook = (jointIndex, parentIndex, maxAngle, rate) => { + const jw = model.jointsWorld; + const jx = jw[jointIndex * 3]; + const jy = jw[jointIndex * 3 + 1]; + const jz = jw[jointIndex * 3 + 2]; + let dx = target.x - jx; + let dy = target.y - jy; + let dz = target.z - jz; + const length = Math.hypot(dx, dy, dz) || 1; + dx /= length; + dy /= length; + dz /= length; + // Direction in the parent joint's frame (rows of R^T are columns of R). + const parentRotation = model.getJointWorldRotation(parentIndex); + const lx = + parentRotation[0] * dx + + parentRotation[3] * dy + + parentRotation[6] * dz; + const ly = + parentRotation[1] * dx + + parentRotation[4] * dy + + parentRotation[7] * dz; + const lz = + parentRotation[2] * dx + + parentRotation[5] * dy + + parentRotation[8] * dz; + // Rotation taking +z to (lx, ly, lz): axis = z × d, angle = atan2. + let axisX = -ly; + let axisY = lx; + const sinAngle = Math.hypot(axisX, axisY); + let angle = Math.atan2(sinAngle, lz); + if (sinAngle > 1e-6) { + axisX /= sinAngle; + axisY /= sinAngle; + } else { + axisX = 0; + axisY = 0; + angle = 0; + } + angle = Math.min(angle, maxAngle); + const targetRx = axisX * angle; + const targetRy = axisY * angle; + const blend = 1 - Math.exp(-rate * dt); + const o = jointIndex * 3; + const smoothed = this._smoothedRotations; + smoothed[o] += (targetRx - smoothed[o]) * blend; + smoothed[o + 1] += (targetRy - smoothed[o + 1]) * blend; + smoothed[o + 2] += (0 - smoothed[o + 2]) * blend; + model.setJointRotation(jointIndex, smoothed[o], smoothed[o + 1], 0); + }; + + if (this.headFollowsCamera) setLook(1, 0, 0.42, 4); + if (this.eyesFollowCamera) { + setLook(2, 1, 0.55, 12); + setLook(3, 1, 0.55, 12); + } + } + + _updateIdleSway(time) { + const sway = 0.05; + this.model.setJointRotation( + 0, + Math.sin(time * 0.53) * sway * 0.6, + Math.sin(time * 0.31) * sway, + Math.sin(time * 0.43) * sway * 0.35 + ); + } + + // --------------------------------------------------------------- update -- + + update() { + const dt = + Math.min(xb.core?.timer?.getDelta?.() ?? 0.016, 0.1) * + this.animationSpeed || 0.016; + this._time += dt; + + if (this.turntable) this.anchor.rotation.y += dt * 0.6; + if (this.tour) this._updateTour(dt); + if (this.morph) this._updateMorph(dt); + if (this._idFade) { + this._idFade.t += dt / 1.1; + this.model.setIdentityBlend(smoothstep(Math.min(this._idFade.t, 1))); + if (this._idFade.t >= 1) this._idFade = null; + } + if (this._exprFade) { + this._exprFade.t += dt / 0.7; + this.model.setExpressionBlend(smoothstep(Math.min(this._exprFade.t, 1))); + if (this._exprFade.t >= 1) this._exprFade = null; + } + this._updatePulse(dt); + if (this.idleSway) this._updateIdleSway(this._time); + if (this.eyesFollowCamera || this.headFollowsCamera) this._updateGaze(dt); + + this.spatialUI?.update(); + if (this.model.dirty) this._refreshGeometry(); + } + + _refreshGeometry() { + const start = performance.now(); + this.model.computeVertices(this.positionAttribute.array); + this.positionAttribute.needsUpdate = true; + this.geometry.computeVertexNormals(); + this.geometry.boundingSphere = null; + if (this.landmarkMesh.visible) this._updateLandmarks(); + if (this.skeletonGroup.visible) this._updateSkeleton(); + this.lastComputeMs = performance.now() - start; + } + + _updateLandmarks() { + const count = this.model.computeLandmarks( + this.positionAttribute.array, + this.landmarkPoints + ); + const matrix = this._tmpMatrix ?? (this._tmpMatrix = new THREE.Matrix4()); + for (let l = 0; l < count; ++l) { + matrix.makeTranslation( + this.landmarkPoints[l * 3], + this.landmarkPoints[l * 3 + 1], + this.landmarkPoints[l * 3 + 2] + ); + this.landmarkMesh.setMatrixAt(l, matrix); + } + this.landmarkMesh.instanceMatrix.needsUpdate = true; + } + + _updateSkeleton() { + const model = this.model; + const joints = model.jointsWorld; + const matrix = this._tmpMatrix ?? (this._tmpMatrix = new THREE.Matrix4()); + for (let j = 0; j < model.numJoints; ++j) { + matrix.makeTranslation( + joints[j * 3], + joints[j * 3 + 1], + joints[j * 3 + 2] + ); + this.jointMesh.setMatrixAt(j, matrix); + } + this.jointMesh.instanceMatrix.needsUpdate = true; + const bones = this.boneAttribute.array; + let b = 0; + for (let j = 1; j < model.numJoints; ++j) { + const p = model.jointParents[j]; + bones[b++] = joints[p * 3]; + bones[b++] = joints[p * 3 + 1]; + bones[b++] = joints[p * 3 + 2]; + bones[b++] = joints[j * 3]; + bones[b++] = joints[j * 3 + 1]; + bones[b++] = joints[j * 3 + 2]; + } + this.boneAttribute.needsUpdate = true; + } + + // ---------------------------------------------------------------- export -- + + exportOBJ() { + const positions = this.positionAttribute.array; + const normals = this.geometry.getAttribute('normal').array; + const triangles = this.model.triangles; + const lines = ['# GNM Head — exported from the XR Blocks GNM demo']; + for (let v = 0; v < positions.length; v += 3) { + lines.push( + `v ${positions[v].toFixed(6)} ${positions[v + 1].toFixed(6)} ` + + `${positions[v + 2].toFixed(6)}` + ); + } + for (let v = 0; v < normals.length; v += 3) { + lines.push( + `vn ${normals[v].toFixed(4)} ${normals[v + 1].toFixed(4)} ` + + `${normals[v + 2].toFixed(4)}` + ); + } + for (let t = 0; t < triangles.length; t += 3) { + const a = triangles[t] + 1; + const b = triangles[t + 1] + 1; + const c = triangles[t + 2] + 1; + lines.push(`f ${a}//${a} ${b}//${b} ${c}//${c}`); + } + return lines.join('\n'); + } + + get jointLabels() { + return JOINT_LABELS; + } +} + +function smoothstep(t) { + return t * t * (3 - 2 * t); +} diff --git a/gnm/shape/demos/web/GNMSpatialUI.js b/gnm/shape/demos/web/GNMSpatialUI.js new file mode 100644 index 00000000..42a96190 --- /dev/null +++ b/gnm/shape/demos/web/GNMSpatialUI.js @@ -0,0 +1,488 @@ +import * as THREE from 'three'; +import {ManipulationBehavior, UICore, UIIcon, UIPanel, UIText} from 'uiblocks'; + +/** + * GNMSpatialUI — in-headset control surface built with the uiblocks addon + * (flexbox-laid-out spatial cards; see src/addons/uiblocks/SKILL.md). + * + * Three draggable UICards arranged around the head mirror the desktop panel: + * SAMPLE (left) semantic identity recipe + all 20 expression + * classes (paged chips), random blend, neutral. + * MOTION (right, top) gaze tracking and animation driver toggles. + * VIEW (right, bottom) material modes and inspection overlays. + * + * Buttons are composed UIPanel + UIText (uiblocks has no button class) and + * reflect live scene state — also when changed from the DOM panel — via + * update(), called from GNMScene.update(). + */ + +// Design tokens (§6.1 of the uiblocks skill): one density, a small type +// scale, spacing rhythm, two radii, and a restrained hex palette. +const PIXEL_SIZE = 0.0015; +const SURFACE = '#12161f'; +const CONTROL = '#232a38'; +const CONTROL_HOVER = '#31394c'; +const CONTROL_ACTIVE = '#1d4238'; +const ACCENT = '#4fe0ae'; +const TEXT = '#f0f3f8'; +const MUTED = '#93a0b8'; +const STROKE = '#333a4c'; +const RADIUS_CARD = 24; +const RADIUS_CONTROL = 12; + +const CHIPS_PER_PAGE = 8; + +function prettify(name) { + return name.replace(/_/g, ' ').replace(/^./, (c) => c.toUpperCase()); +} + +export class GNMSpatialUI { + /** @param scene The owning GNMScene (an xb.Script). */ + constructor(scene) { + this.scene = scene; + this.model = scene.model; + this.samplers = scene.samplers; + this.uiCore = new UICore(scene); + + this._genderWeights = [1, 1]; + this._ethnicityWeights = [1, 1, 1, 1]; + this._chipPage = 0; + this._chips = []; + this._toggles = []; // {button, get} + this._modeButtons = []; // {button, mode} + this._recipeButtons = {gender: [], ethnicity: []}; + this._lastStates = new Map(); + } + + build() { + this._buildSampleCard(); + this._buildMotionCard(); + this._buildViewCard(); + this._highlightRecipe(); + this._renderChipPage(); + } + + setStatus(text) { + this._statusText?.setText(text || ' '); + } + + // ------------------------------------------------------------ primitives -- + + _card(name, sizeX, sizeY, x, y, z, rotationY) { + return this.uiCore.createCard({ + name, + sizeX, + sizeY, + pixelSize: PIXEL_SIZE, + position: new THREE.Vector3(x, y, z), + rotation: new THREE.Quaternion().setFromEuler( + new THREE.Euler(0, rotationY, 0) + ), + behaviors: [ + new ManipulationBehavior({ + draggable: true, + faceCamera: true, + manipulationMargin: 24, + manipulationCornerRadius: RADIUS_CARD, + }), + ], + }); + } + + _surface(card, {padding = 20, gap = 12} = {}) { + const panel = new UIPanel({ + width: '100%', + height: '100%', + flexDirection: 'column', + fillColor: SURFACE, + cornerRadius: RADIUS_CARD, + padding, + gap, + strokeWidth: 1, + strokeColor: STROKE, + strokeAlign: 'inside', + dropShadowColor: '#000000', + dropShadowBlur: 20, + dropShadowSpread: 2, + }); + card.add(panel); + return panel; + } + + _header(parent, icon, title) { + const row = new UIPanel({ + width: '100%', + flexDirection: 'row', + alignItems: 'center', + gap: 10, + }); + row.add(new UIIcon(icon, {width: 26, height: 26, color: ACCENT})); + row.add(new UIText(title, {fontSize: 24, fontWeight: 'bold', color: TEXT})); + parent.add(row); + return row; + } + + _sectionLabel(parent, label) { + parent.add( + new UIText(label, {fontSize: 14, fontWeight: 'bold', color: MUTED}) + ); + } + + _row(parent, {gap = 10, height} = {}) { + const row = new UIPanel({ + width: '100%', + flexDirection: 'row', + gap, + ...(height ? {height} : {}), + alignItems: 'center', + }); + parent.add(row); + return row; + } + + /** Composes a button from UIPanel + UIText (uiblocks skill §5.3). */ + _button( + parent, + label, + {fontSize = 16, height = 40, accent = false, onClick} + ) { + const state = { + hovered: false, + base: CONTROL, + textColor: accent ? ACCENT : TEXT, + }; + const panel = new UIPanel({ + flexGrow: 1, + height, + cornerRadius: RADIUS_CONTROL, + fillColor: state.base, + justifyContent: 'center', + alignItems: 'center', + padding: 4, + onHoverEnter: () => { + state.hovered = true; + panel.setFillColor(CONTROL_HOVER); + }, + onHoverExit: () => { + state.hovered = false; + panel.setFillColor(state.base); + }, + onClick: () => { + onClick?.(); + return true; + }, + }); + const text = new UIText(label, { + fontSize, + color: state.textColor, + textAlign: 'center', + }); + panel.add(text); + parent.add(panel); + const button = { + panel, + text, + setActive(active) { + state.base = active ? CONTROL_ACTIVE : CONTROL; + text.setColor(active ? ACCENT : state.textColor); + if (!state.hovered) panel.setFillColor(state.base); + }, + setLabel(value) { + text.setText(value); + }, + }; + return button; + } + + // ----------------------------------------------------------------- cards -- + + _buildSampleCard() { + const card = this._card('GNMSample', 0.6, 0.8, -0.62, 1.32, -0.46, 0.6); + const root = this._surface(card); + + this._header(root, 'face', 'GNM Sample'); + this._sectionLabel(root, 'IDENTITY'); + + const genderRow = this._row(root); + this._recipeButtons.gender = [ + ['Female', 0], + ['Male', 1], + ['Mix', -1], + ].map(([label, index]) => + this._button(genderRow, label, { + fontSize: 15, + height: 36, + onClick: () => this._pickGender(index), + }) + ); + + const ethnicityRow = this._row(root, {gap: 8}); + this._recipeButtons.ethnicity = [ + ['Mid-East', 0], + ['Asian', 1], + ['White', 2], + ['Black', 3], + ['Mix', -1], + ].map(([label, index]) => + this._button(ethnicityRow, label, { + fontSize: 12, + height: 36, + onClick: () => this._pickEthnicity(index), + }) + ); + + const identityActions = this._row(root); + this._button(identityActions, 'New face', { + accent: true, + onClick: () => this._sampleIdentity(), + }); + this._button(identityActions, 'Template', { + onClick: () => { + this.model.resetIdentity(); + this.scene.onModelChanged?.(); + this.scene._emitStatus('template (mean) face'); + }, + }); + + this._sectionLabel(root, 'EXPRESSION'); + + const classes = this.samplers.expressionClasses; + for (let r = 0; r < CHIPS_PER_PAGE / 4; ++r) { + const chipRow = this._row(root, {gap: 8}); + for (let c = 0; c < 4; ++c) { + const chip = this._button(chipRow, '', { + fontSize: 12, + height: 40, + onClick: () => { + if (chip.classIndex === undefined || chip.classIndex < 0) return; + this.scene.sampleExpression(chip.classIndex, 0.9); + this.scene._emitStatus( + `expression: ${classes[chip.classIndex].replace(/_/g, ' ')}` + ); + }, + }); + this._chips.push(chip); + } + } + + const pageCount = Math.ceil(classes.length / CHIPS_PER_PAGE); + const pager = this._row(root); + this._button(pager, '<', { + fontSize: 15, + height: 30, + onClick: () => { + this._chipPage = (this._chipPage + pageCount - 1) % pageCount; + this._renderChipPage(); + }, + }); + this._pageText = new UIText('', { + fontSize: 14, + color: MUTED, + textAlign: 'center', + flexGrow: 1, + }); + pager.add(this._pageText); + this._button(pager, '>', { + fontSize: 15, + height: 30, + onClick: () => { + this._chipPage = (this._chipPage + 1) % pageCount; + this._renderChipPage(); + }, + }); + + const expressionActions = this._row(root); + this._button(expressionActions, 'Random blend', { + accent: true, + onClick: () => { + this.scene.sampleRandomExpression(0.9); + this.scene._emitStatus('random expression blend'); + }, + }); + this._button(expressionActions, 'Neutral', { + onClick: () => { + this.model.resetExpression(); + this.scene.onModelChanged?.(); + this.scene._emitStatus('neutral expression'); + }, + }); + + this._statusText = new UIText(' ', { + fontSize: 14, + color: ACCENT, + textAlign: 'center', + width: '100%', + }); + root.add(this._statusText); + } + + _buildMotionCard() { + const card = this._card('GNMMotion', 0.52, 0.46, 0.62, 1.52, -0.46, -0.6); + const root = this._surface(card); + this._header(root, 'animation', 'Motion'); + + const scene = this.scene; + const specs = [ + [ + 'Eyes follow', + () => scene.eyesFollowCamera, + (v) => (scene.eyesFollowCamera = v), + ], + [ + 'Head follow', + () => scene.headFollowsCamera, + (v) => (scene.headFollowsCamera = v), + ], + ['Tour', () => !!scene.tour, (v) => scene.setExpressionTour(v)], + ['Morph', () => !!scene.morph, (v) => scene.setIdentityMorph(v)], + ['Sway', () => scene.idleSway, (v) => (scene.idleSway = v)], + ['Turntable', () => scene.turntable, (v) => (scene.turntable = v)], + ]; + for (let r = 0; r < specs.length; r += 2) { + const row = this._row(root); + for (const [label, get, set] of specs.slice(r, r + 2)) { + const button = this._button(row, label, { + fontSize: 15, + onClick: () => set(!get()), + }); + this._toggles.push({button, get}); + } + } + + const actions = this._row(root); + this._button(actions, 'Reset pose', { + onClick: () => { + this.model.resetPose(); + scene._smoothedRotations.fill(0); + scene.onModelChanged?.(); + }, + }); + this._button(actions, 'Neutral all', { + onClick: () => scene.resetToNeutral(), + }); + } + + _buildViewCard() { + const card = this._card('GNMView', 0.52, 0.4, 0.64, 1.08, -0.46, -0.6); + const root = this._surface(card); + this._header(root, 'visibility', 'View'); + + const scene = this.scene; + const modeRow = this._row(root, {gap: 8}); + for (const [mode, label] of [ + ['studio', 'Studio'], + ['clay', 'Clay'], + ['normals', 'Normals'], + ['regions', 'Regions'], + ]) { + const button = this._button(modeRow, label, { + fontSize: 13, + height: 36, + onClick: () => scene.setMaterialMode(mode), + }); + this._modeButtons.push({button, mode}); + } + + const overlaySpecs = [ + [ + 'Wireframe', + () => scene.wireframe.visible, + (v) => scene.setWireframeVisible(v), + ], + [ + 'Landmarks', + () => scene.landmarkMesh.visible, + (v) => scene.setLandmarksVisible(v), + ], + [ + 'Skeleton', + () => scene.skeletonGroup.visible, + (v) => scene.setSkeletonVisible(v), + ], + [ + 'Anatomy', + () => !scene.visibleComponents[0], + (v) => scene.setComponentVisible(0, !v), + ], + ]; + for (let r = 0; r < overlaySpecs.length; r += 2) { + const row = this._row(root); + for (const [label, get, set] of overlaySpecs.slice(r, r + 2)) { + const button = this._button(row, label, { + fontSize: 15, + onClick: () => set(!get()), + }); + this._toggles.push({button, get}); + } + } + } + + // -------------------------------------------------------------- behavior -- + + _pickGender(index) { + this._genderWeights = + index < 0 ? [1, 1] : [index === 0 ? 1 : 0, index === 1 ? 1 : 0]; + this._highlightRecipe(); + this._sampleIdentity(); + } + + _pickEthnicity(index) { + this._ethnicityWeights = + index < 0 ? [1, 1, 1, 1] : [0, 1, 2, 3].map((i) => (i === index ? 1 : 0)); + this._highlightRecipe(); + this._sampleIdentity(); + } + + _sampleIdentity() { + this.scene.sampleIdentity(this._genderWeights, this._ethnicityWeights, 0.9); + this.scene._emitStatus('sampled identity'); + } + + _highlightRecipe() { + const [female, male] = this._genderWeights; + const genderActive = [ + female === 1 && male === 0, + male === 1 && female === 0, + female === male, + ]; + this._recipeButtons.gender.forEach((button, i) => + button.setActive(genderActive[i]) + ); + const mix = this._ethnicityWeights.every((w) => w === 1); + this._recipeButtons.ethnicity.forEach((button, i) => { + const active = i === 4 ? mix : !mix && this._ethnicityWeights[i] === 1; + button.setActive(active); + }); + } + + _renderChipPage() { + const classes = this.samplers.expressionClasses; + const pageCount = Math.ceil(classes.length / CHIPS_PER_PAGE); + this._chips.forEach((chip, i) => { + const classIndex = this._chipPage * CHIPS_PER_PAGE + i; + chip.classIndex = classIndex < classes.length ? classIndex : -1; + chip.setLabel( + chip.classIndex >= 0 ? prettify(classes[chip.classIndex]) : '' + ); + }); + this._pageText?.setText(`${this._chipPage + 1} / ${pageCount}`); + } + + /** Reflects scene state into toggle fills and mode highlights. */ + update() { + for (const toggle of this._toggles) { + const on = !!toggle.get(); + if (this._lastStates.get(toggle) !== on) { + this._lastStates.set(toggle, on); + toggle.button.setActive(on); + } + } + const mode = this.scene.materialMode; + if (this._lastStates.get('mode') !== mode) { + this._lastStates.set('mode', mode); + for (const entry of this._modeButtons) { + entry.button.setActive(entry.mode === mode); + } + } + } +} diff --git a/gnm/shape/demos/web/README.md b/gnm/shape/demos/web/README.md new file mode 100644 index 00000000..6c47d3c3 --- /dev/null +++ b/gnm/shape/demos/web/README.md @@ -0,0 +1,36 @@ +# GNM Web Demo + +[![GNM Head Explorer Preview](teaser/teaser.webm)](https://xrblocks.github.io/demos/gnm/) + +**Live Demo:** [https://xrblocks.github.io/demos/gnm/](https://xrblocks.github.io/demos/gnm/) + +This is a standalone web demonstration of the [GNM](https://github.com/google/gnm) (Generative aNthropometric Model) running purely in the Chrome browser +across desktop, mobile, and [Android XR](https://android.com/xr) devices. + +The demo runs using [XR Blocks](https://github.com/google/xrblocks), loading it dynamically via CDN, which provides the 3D scene, rendering, and spatial UI. You can explore more interactive XR examples and the full SDK in the [XR Blocks repository](https://github.com/google/xrblocks). + +## Running the Demo + +Because this demo uses ES modules, you must serve it over HTTP (opening `index.html` directly from your file system via `file://` will not work). + +You can use any local web server. For example, if you have Python installed, run this command in this directory: + +```bash +# Python 3 +python -m http.server 8000 +``` + +Then, open your browser and navigate to: +[http://localhost:8000/](http://localhost:8000/) + +## Model Assets + +By default, the demo fetches the required model weights (`gnm_head_web.bin` and `gnm_samplers_web.bin`) dynamically from a CDN. + +If you prefer to run the demo entirely offline, you can download the pre-converted `.bin` files from the [assets-gnm repository](https://github.com/xrblocks/assets-gnm/tree/main). + +1. Create an `assets/` folder in this directory. +2. Place `gnm_head_web.bin` and `gnm_samplers_web.bin` inside `assets/`. +3. Open `main.js` and set `const USE_LOCAL_ASSETS = true;`, or append `?localAssets=1` to your URL. + +Alternatively, you can generate the `.bin` files yourself from the main GNM repository using the script located in `tools/export_gnm_web.py`. diff --git a/gnm/shape/demos/web/SemanticSampler.js b/gnm/shape/demos/web/SemanticSampler.js new file mode 100644 index 00000000..0393c9f5 --- /dev/null +++ b/gnm/shape/demos/web/SemanticSampler.js @@ -0,0 +1,181 @@ +/** + * SemanticSampler.js — JavaScript port of gnm/shape/semantic_sampler.py. + * + * The identity and expression samplers are conditional-VAE decoders: a small + * MLP mapping [latent z, one-hot condition] to a GNM parameter vector. The + * exported weights (tools/export_gnm_web.py) run here as plain typed-array + * matrix products, enabling live semantic sampling in the browser: + * + * identity: z(64) + [gender(2), ethnicity(4)] -> 253 identity params + * expression: z(64) + [expression class(20)] -> 383 expression params + * + * Blending mirrors the Python reference: expressions blend per-class latents + * and one-hots; identities share one latent with blended condition one-hots. + */ + +import {fetchWithProgress, parseContainer} from './GNMModel.js'; + +/** Deterministic 32-bit PRNG (mulberry32) so results are reproducible. */ +export function createRng(seed) { + let state = seed >>> 0; + return function () { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +/** Standard normal via Box–Muller. */ +function gaussian(rng) { + let u = 0; + while (u === 0) u = rng(); + const v = rng(); + return Math.sqrt(-2 * Math.log(u)) * Math.cos(2 * Math.PI * v); +} + +class MLPDecoder { + /** @param layers Array of {weights, bias, activation, inputDim, outputDim} */ + constructor(layers) { + this.layers = layers; + this.inputDim = layers[0].inputDim; + this.outputDim = layers[layers.length - 1].outputDim; + let maxDim = this.inputDim; + for (const layer of layers) maxDim = Math.max(maxDim, layer.outputDim); + this._bufferA = new Float32Array(maxDim); + this._bufferB = new Float32Array(maxDim); + } + + /** input: Float32Array(inputDim) -> Float32Array(outputDim) (fresh copy). */ + forward(input) { + let x = this._bufferA; + let y = this._bufferB; + x.set(input); + let currentDim = this.inputDim; + for (const layer of this.layers) { + const {weights, bias, activation, outputDim} = layer; + for (let o = 0; o < outputDim; ++o) { + let acc = bias[o]; + for (let i = 0; i < currentDim; ++i) { + acc += x[i] * weights[i * outputDim + o]; + } + y[o] = activation === 'relu' && acc < 0 ? 0 : acc; + } + [x, y] = [y, x]; + currentDim = outputDim; + } + return x.slice(0, currentDim); + } +} + +export class GNMSamplers { + constructor(meta, sections) { + this.identityMeta = meta.identity; + this.expressionMeta = meta.expression; + this.identityDecoder = new MLPDecoder( + meta.identity.layers.map((l) => ({ + ...l, + weights: sections[l.weights], + bias: sections[l.bias], + })) + ); + this.expressionDecoder = new MLPDecoder( + meta.expression.layers.map((l) => ({ + ...l, + weights: sections[l.weights], + bias: sections[l.bias], + })) + ); + this.genders = meta.identity.genders; + this.ethnicities = meta.identity.ethnicities; + this.expressionClasses = meta.expression.labels; + this._rng = createRng(20260717); + } + + static async load(url, onProgress) { + const buffer = await fetchWithProgress(url, onProgress); + const {meta, sections} = parseContainer(buffer); + return new GNMSamplers(meta, sections); + } + + /** Reseeds the internal RNG (useful for reproducible demos). */ + seed(value) { + this._rng = createRng(value); + } + + _sampleLatent(dim, sigma) { + const z = new Float32Array(dim); + for (let i = 0; i < dim; ++i) z[i] = gaussian(this._rng) * sigma; + return z; + } + + /** + * Samples an identity vector. + * genderWeights / ethnicityWeights: arrays of non-negative weights (one per + * class, normalized internally); pass a one-hot for a pure class. + * sigma scales latent variation (0 = class mean). + */ + sampleIdentity(genderWeights, ethnicityWeights, sigma = 1) { + const {latentDim} = this.identityMeta; + const input = new Float32Array(this.identityDecoder.inputDim); + input.set(this._sampleLatent(latentDim, sigma), 0); + normalizeInto(input, latentDim, genderWeights); + normalizeInto(input, latentDim + genderWeights.length, ethnicityWeights); + return this.identityDecoder.forward(input); + } + + /** + * Samples an expression vector for one class index, or blends several: + * pass a Map/object of classIndex -> weight. Each class contributes its own + * latent sample, matching ExpressionSampler.blend_expressions. + */ + sampleExpression(classWeights, sigma = 1) { + const {latentDim, conditionDim} = this.expressionMeta; + const entries = + typeof classWeights === 'number' + ? [[classWeights, 1]] + : Object.entries(classWeights).map(([k, w]) => [Number(k), w]); + let total = 0; + for (const [, w] of entries) total += w; + if (!(total > 0)) total = 1; + + const input = new Float32Array(this.expressionDecoder.inputDim); + for (const [classIndex, weight] of entries) { + const w = weight / total; + if (w === 0) continue; + const z = this._sampleLatent(latentDim, sigma); + for (let i = 0; i < latentDim; ++i) input[i] += z[i] * w; + input[latentDim + classIndex] += w; + } + return this.expressionDecoder.forward(input); + } + + /** Random blended identity, mirroring IdentitySampler.randomize_identities. */ + randomIdentity(sigma = 1) { + const genderWeights = this.genders.map(() => this._rng()); + const ethnicityWeights = this.ethnicities.map(() => this._rng()); + return this.sampleIdentity(genderWeights, ethnicityWeights, sigma); + } + + /** Random 2–3 class expression blend, mirroring randomize_expressions. */ + randomExpression(sigma = 1, maxClasses = 3) { + const count = 2 + Math.floor(this._rng() * (maxClasses - 1)); + const chosen = new Set(); + while (chosen.size < count) { + chosen.add(Math.floor(this._rng() * this.expressionClasses.length)); + } + const weights = {}; + for (const c of chosen) weights[c] = this._rng(); + return this.sampleExpression(weights, sigma); + } +} + +function normalizeInto(target, offset, weights) { + let total = 0; + for (const w of weights) total += w; + if (!(total > 0)) total = 1; + for (let i = 0; i < weights.length; ++i) { + target[offset + i] = weights[i] / total; + } +} diff --git a/gnm/shape/demos/web/demo.css b/gnm/shape/demos/web/demo.css new file mode 100644 index 00000000..315ba759 --- /dev/null +++ b/gnm/shape/demos/web/demo.css @@ -0,0 +1,236 @@ +html, +body { + height: 100%; + margin: 0; + padding: 0; + overflow: hidden; /* Prevents scroll bars if the image overflows slightly */ +} + +body { + background-size: cover; + background-position: center; + background-repeat: no-repeat; + color: #fff; + font-family: Monospace; + font-size: 13px; + line-height: 24px; + overscroll-behavior: none; +} + +.content { + position: relative; + color: #fff; + font-family: Monospace; + font-size: 13px; + line-height: 24px; + overscroll-behavior: none; +} + +a { + color: #ff0; + text-decoration: none; +} + +a:hover { + text-decoration: underline; +} + +button { + cursor: pointer; + text-transform: uppercase; +} + +#info { + position: absolute; + top: 0px; + width: 100%; + padding: 10px; + box-sizing: border-box; + text-align: center; + -moz-user-select: none; + -webkit-user-select: none; + -ms-user-select: none; + user-select: none; + pointer-events: none; + z-index: 1; /* TODO Solve this in HTML */ +} + +a, +button, +input, +select { + pointer-events: auto; +} + +.lil-gui { + z-index: 2 !important; /* TODO Solve this in HTML */ +} + +@media all and (max-width: 640px) { + .lil-gui.root { + right: auto; + top: auto; + max-height: 50%; + max-width: 80%; + bottom: 0; + left: 0; + } +} + +#overlay { + position: absolute; + font-size: 16px; + z-index: 2; + top: 0; + left: 0; + width: 100%; + height: 100%; + display: flex; + align-items: center; + justify-content: center; + flex-direction: column; + background: rgba(0, 0, 0, 0.7); +} + +#overlay button { + background: transparent; + border: 0; + border: 1px solid rgb(255, 255, 255); + border-radius: 4px; + color: #ffffff; + padding: 12px 18px; + text-transform: uppercase; + cursor: pointer; +} + +#notSupported { + width: 50%; + margin: auto; + background-color: #f00; + margin-top: 20px; + padding: 10px; +} + +header { + position: fixed; + top: 0px; + left: 0px; + width: 100%; + padding: 0.8rem 1.6rem; + color: rgb(255, 255, 255); + mix-blend-mode: exclusion; + display: flex; + align-items: center; + justify-content: space-between; + z-index: 101; + pointer-events: none; +} + +.logo { + font-family: 'Google Sans Text', system-ui, sans-serif; + font-size: 1.8rem; + font-style: normal; + font-weight: 500; + letter-spacing: -0.018rem; + transform: translate(0px, 0.05rem); +} + +.logo-link { + display: flex; + align-items: center; + gap: 0.4rem; + pointer-events: all; + flex: 0.55 1 0%; +} + +.logo-text { + font-family: 'Google Sans Text', system-ui, sans-serif; + color: white; + font-size: 1.8rem; + font-style: normal; + font-weight: 500; + letter-spacing: -0.018rem; + transform: translate(0px, 0.05rem); +} + +.logo-icon { + color: white; + width: 1.6rem; + height: 1.6rem; + margin-left: 0.2rem; + margin-right: 0.2rem; +} + +.slogan { + flex: 0.25 1 0%; + font-family: 'Google Sans Mono', system-ui, sans-serif; + font-weight: 400; + font-size: 1rem; + line-height: 120%; + letter-spacing: 0.06rem; + display: block; +} + +#XRButtonWrapper { + display: flex; + flex-direction: column; + justify-content: center; + background-color: transparent; + color: white; + padding: 10px 20px; + border: none; + cursor: pointer; + font-size: 2rem; + border-radius: 1rem; + display: flex; + align-items: center; + position: absolute; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + width: 30rem; + height: 16rem; + gap: 1rem; +} + +.XRButton { + color: white; + background-color: #e74c3c; + border: 0; + flex-grow: 1; + font: 'normal sans-serif'; + font-size: 2rem; + width: 100%; + cursor: pointer; + border-radius: 3rem; +} + +.XRButton svg { + margin-right: 8px; /* Space between the icon and text */ +} + +.XRButton:hover { + background-color: #c0392b; /* Darken the background on hover */ +} + +#xrlogo { + display: inline-block; + width: 3rem; + height: 3rem; + margin-right: 0.8rem; + background-image: url('xrlogo.webp'); + background-size: contain; /* Ensures the image fits within the defined size */ + background-repeat: no-repeat; + vertical-align: middle; /* Aligns with the text */ +} + +.background-image { + position: fixed; + top: 0; + left: 0; + width: 100vw; + height: 100vh; + z-index: -1; /* Sends the background behind the content */ + background-size: cover; /* Keeps aspect ratio and covers the screen */ + background-position: center; +} diff --git a/gnm/shape/demos/web/gnm.css b/gnm/shape/demos/web/gnm.css new file mode 100644 index 00000000..c2f00240 --- /dev/null +++ b/gnm/shape/demos/web/gnm.css @@ -0,0 +1,341 @@ +/* GNM Head Explorer — control panel styling. */ + +#gnm-panel { + position: absolute; + top: 12px; + right: 12px; + width: 340px; + max-height: calc(100% - 24px); + display: flex; + flex-direction: column; + z-index: 5; + color: #e8ecf4; + color-scheme: dark; /* dark native widgets: select popups, sliders, checks */ + font-family: + 'Inter', + system-ui, + -apple-system, + sans-serif; + font-size: 12.5px; + line-height: 1.45; + background: rgba(13, 17, 26, 0.86); + backdrop-filter: blur(14px); + -webkit-backdrop-filter: blur(14px); + border: 1px solid rgba(255, 255, 255, 0.09); + border-radius: 14px; + box-shadow: 0 12px 40px rgba(0, 0, 0, 0.45); + overflow: hidden; + user-select: none; +} + +#gnm-panel.collapsed { + width: 230px; +} + +#gnm-panel.collapsed #gnm-tabs, +#gnm-panel.collapsed #gnm-pages, +#gnm-panel.collapsed #gnm-status, +#gnm-panel.collapsed .gnm-footer, +#gnm-panel.collapsed .gnm-header p { + display: none; +} + +#gnm-panel.collapsed .gnm-header { + padding: 8px 12px; +} + +#gnm-panel .gnm-header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 12px 14px 8px; +} + +#gnm-panel .gnm-header h1 { + margin: 0; + font-size: 15px; + font-weight: 650; + letter-spacing: 0.02em; + color: #e8ecf4; +} + +#gnm-panel .gnm-header p { + margin: 1px 0 0; + font-size: 11px; + color: #93a0b8; +} + +#gnm-collapse { + width: 26px; + height: 26px; + border-radius: 8px; + border: 1px solid rgba(255, 255, 255, 0.14); + background: rgba(255, 255, 255, 0.06); + color: #e8ecf4; + font-size: 15px; + line-height: 1; + text-transform: none; +} + +#gnm-status { + min-height: 16px; + padding: 0 14px 6px; + font-size: 11px; + color: #4fe0ae; +} + +#gnm-tabs { + display: flex; + gap: 2px; + padding: 0 10px; + border-bottom: 1px solid rgba(255, 255, 255, 0.08); +} + +#gnm-tabs button { + flex: 1; + padding: 7px 2px; + font-size: 11px; + border: none; + border-bottom: 2px solid transparent; + background: none; + color: #93a0b8; + text-transform: none; +} + +#gnm-tabs button.active { + color: #ffffff; + border-bottom-color: #4f8ce0; +} + +#gnm-pages { + overflow-y: auto; + flex: 1; + scrollbar-width: thin; + scrollbar-color: #3a4356 transparent; +} + +.gnm-page { + display: none; + padding: 10px 14px; +} + +.gnm-page.active { + display: block; +} + +.gnm-page section { + margin-bottom: 14px; +} + +.gnm-page h2 { + margin: 0 0 6px; + font-size: 11px; + font-weight: 650; + text-transform: uppercase; + letter-spacing: 0.08em; + color: #8fa0bd; +} + +.gnm-btn-row { + display: flex; + flex-wrap: wrap; + gap: 6px; + margin: 6px 0; +} + +.gnm-btn { + padding: 6px 10px; + border-radius: 8px; + border: 1px solid rgba(255, 255, 255, 0.14); + background: rgba(79, 140, 224, 0.16); + color: #e8ecf4; + font-size: 11.5px; + text-transform: none; +} + +.gnm-btn:hover { + background: rgba(79, 140, 224, 0.34); +} + +.gnm-btn.active { + background: #4f8ce0; + border-color: #4f8ce0; + color: #fff; +} + +.gnm-btn.ghost { + background: none; + border-style: dashed; + color: #93a0b8; + width: 100%; + margin-top: 4px; +} + +.gnm-chips { + display: flex; + flex-wrap: wrap; + gap: 5px; +} + +.gnm-chip { + padding: 4px 9px; + border-radius: 999px; + border: 1px solid rgba(255, 255, 255, 0.16); + background: rgba(255, 255, 255, 0.05); + color: #cfd8e8; + font-size: 11px; + text-transform: none; +} + +.gnm-chip:hover { + background: rgba(79, 224, 174, 0.22); + border-color: rgba(79, 224, 174, 0.6); +} + +.gnm-slider { + display: grid; + grid-template-columns: 64px 1fr 38px; + align-items: center; + gap: 8px; + padding: 2px 0; +} + +.gnm-slider.wide { + grid-template-columns: 64px 1fr 38px; +} + +.gnm-slider span { + color: #aab6ca; + font-size: 11px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.gnm-slider output { + font-size: 10.5px; + color: #7f8ca3; + text-align: right; + font-variant-numeric: tabular-nums; +} + +.gnm-slider input[type='range'] { + width: 100%; + height: 4px; + accent-color: #4f8ce0; +} + +.gnm-slider input[type='range']:disabled { + opacity: 0.35; +} + +.gnm-toggle { + display: flex; + align-items: center; + gap: 8px; + padding: 3px 0; + cursor: pointer; +} + +.gnm-toggle input { + accent-color: #4f8ce0; +} + +.gnm-selects { + display: flex; + gap: 10px; + margin-bottom: 6px; +} + +.gnm-selects label { + display: flex; + flex-direction: column; + gap: 3px; + flex: 1; + font-size: 11px; + color: #93a0b8; +} + +.gnm-selects select { + background: rgba(255, 255, 255, 0.07); + color: #e8ecf4; + border: 1px solid rgba(255, 255, 255, 0.14); + border-radius: 7px; + padding: 5px 6px; + font-size: 12px; +} + +.gnm-selects select option { + background: #131826; + color: #e8ecf4; +} + +.gnm-hint { + font-size: 11px; + color: #7f8ca3; + font-style: italic; +} + +#gnm-panel .gnm-footer { + padding: 8px 14px 10px; + border-top: 1px solid rgba(255, 255, 255, 0.08); + font-size: 10.5px; + color: #7f8ca3; +} + +#gnm-panel .gnm-footer a { + color: #4fa3e0; +} + +/* Loading overlay */ +#gnm-loading { + position: absolute; + inset: 0; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 14px; + z-index: 20; + background: radial-gradient(circle at 50% 40%, #141a26, #07090e); + color: #e8ecf4; + font-family: 'Inter', system-ui, sans-serif; +} + +#gnm-loading h1 { + margin: 0; + font-size: 22px; + font-weight: 650; +} + +#gnm-loading p { + margin: 0; + color: #93a0b8; + font-size: 13px; +} + +#gnm-loading .bar { + width: 260px; + height: 6px; + border-radius: 3px; + background: rgba(255, 255, 255, 0.1); + overflow: hidden; +} + +#gnm-loading .bar div { + height: 100%; + width: 0%; + border-radius: 3px; + background: linear-gradient(90deg, #4f8ce0, #4fe0ae); + transition: width 0.15s ease-out; +} + +@media (max-width: 720px) { + #gnm-panel { + width: min(92vw, 340px); + max-height: 55%; + top: auto; + bottom: 10px; + right: 4vw; + } +} diff --git a/gnm/shape/demos/web/index.html b/gnm/shape/demos/web/index.html new file mode 100644 index 00000000..89facdb0 --- /dev/null +++ b/gnm/shape/demos/web/index.html @@ -0,0 +1,55 @@ + + + + GNM Head Explorer | XR Blocks + + + + + + + + + + +
+

GNM Head Explorer

+
+

Preparing…

+
+ + + diff --git a/gnm/shape/demos/web/main.js b/gnm/shape/demos/web/main.js new file mode 100644 index 00000000..3f0e64e7 --- /dev/null +++ b/gnm/shape/demos/web/main.js @@ -0,0 +1,85 @@ +import 'xrblocks/addons/simulator/SimulatorAddons.js'; + +import * as uikit from '@pmndrs/uikit'; +import * as xb from 'xrblocks'; + +import {GNMControls} from './GNMControls.js'; +import {GNMHeadModel} from './GNMModel.js'; +import {GNMScene} from './GNMScene.js'; +import {GNMSamplers} from './SemanticSampler.js'; + +// The GNM model weights are hosted on a CDN (pinned to a commit) rather than +// checked into the repo, since the head model alone is ~35 MB. They are the +// web export of gnm/shape/data (see tools/export_gnm_web.py). +const CDN_ASSETS_BASE = + 'https://rawcdn.githack.com/xrblocks/assets-gnm/8480138a42ae746a2f7c9808a51ef23af7648653'; +// Local copies for offline development: generate them from a GNM checkout with +// python tools/export_gnm_web.py --gnm_root=/path/to/gnm --out_dir=./assets +// then load them with the toggle below. +const LOCAL_ASSETS_BASE = './assets'; + +// DEBUG TOGGLE — flip to true to load the model from local ./assets instead of +// the public CDN. Can also be overridden per-load without editing code via the +// URL query: ?localAssets=1 (local) or ?localAssets=0 (CDN). +const USE_LOCAL_ASSETS = false; + +const useLocalAssets = xb.getUrlParamBool('localAssets', USE_LOCAL_ASSETS); +const ASSETS_BASE = useLocalAssets ? LOCAL_ASSETS_BASE : CDN_ASSETS_BASE; +const HEAD_MODEL_URL = `${ASSETS_BASE}/gnm_head_web.bin`; +const SAMPLERS_URL = `${ASSETS_BASE}/gnm_samplers_web.bin`; +console.info( + `[GNM] loading model assets from ${useLocalAssets ? 'LOCAL' : 'CDN'}: ${ASSETS_BASE}` +); + +function setLoadingProgress(fraction, label) { + const bar = document.querySelector('#gnm-loading .bar div'); + const text = document.querySelector('#gnm-loading p'); + if (bar) bar.style.width = `${Math.round(fraction * 100)}%`; + if (label && text) text.textContent = label; +} + +async function start() { + try { + let modelProgress = 0; + let samplerProgress = 0; + const report = () => + setLoadingProgress( + modelProgress * 0.92 + samplerProgress * 0.08, + 'Downloading GNM model data…' + ); + const [model, samplers] = await Promise.all([ + GNMHeadModel.load(HEAD_MODEL_URL, (p) => { + modelProgress = p; + report(); + }), + GNMSamplers.load(SAMPLERS_URL, (p) => { + samplerProgress = p; + report(); + }), + ]); + setLoadingProgress(1, 'Starting XR Blocks…'); + + const options = new xb.Options(); + options.enableUI(); + options.uikit.enable(uikit); // registers the uikit renderer for uiblocks + options.enableReticles(); + options.setAppTitle('GNM Head Explorer'); + options.xrButton.startText = ' EXPLORE IN XR'; + options.xrButton.endText = ' EXIT XR'; + + const scene = new GNMScene(model, samplers); + xb.add(scene); + window.gnm = {model, samplers, scene}; + + const controls = new GNMControls(model, samplers, scene); + controls.attach(); + document.getElementById('gnm-loading')?.remove(); + + await xb.init(options); + } catch (error) { + setLoadingProgress(0, `Failed to load: ${error.message}`); + console.error(error); + } +} + +document.addEventListener('DOMContentLoaded', start); diff --git a/gnm/shape/demos/web/teaser/teaser.webm b/gnm/shape/demos/web/teaser/teaser.webm new file mode 100644 index 00000000..88177c06 Binary files /dev/null and b/gnm/shape/demos/web/teaser/teaser.webm differ diff --git a/gnm/shape/demos/web/tools/export_gnm_web.py b/gnm/shape/demos/web/tools/export_gnm_web.py new file mode 100644 index 00000000..e285fa06 --- /dev/null +++ b/gnm/shape/demos/web/tools/export_gnm_web.py @@ -0,0 +1,338 @@ +"""Exports the GNM head model + semantic samplers to compact web binaries. + +Reads the GNM npz model, the 68-landmark definition, and the two Keras CVAE +decoder .h5 files, and writes two container files consumed by the XR Blocks +GNM demo: + + assets/gnm_head_web.bin quantized model (bases as int8 + f32 scales) + assets/gnm_samplers_web.bin identity/expression decoder MLP weights (f32) + +Container layout (little endian): + bytes 0..3 magic 'GNMW' + uint32 format version (1) + uint32 JSON header byte length + ... JSON header (utf-8) + ... binary sections, each 4-byte aligned, described by the header + +Usage: + python export_gnm_web.py --gnm_root=path/to/gnm_repo --out_dir=../assets + +Requires only numpy + h5py (no TensorFlow). +""" + +import argparse +import json +import os +import re + +import h5py +import numpy as np + +# Class label names mirrored from gnm/shape/semantic_sampler.py. +GENDERS = ['female', 'male'] +ETHNICITIES = ['middle_eastern', 'asian', 'white', 'black'] +EXPRESSION_CLASSES = [ + 'surprise', 'disgust', 'suck', 'compress_face', 'stretch_face', 'happy', + 'squint', 'platysma', 'blow', 'funneler', 'smile_wide', 'corners_down', + 'pucker', 'wink_left', 'wink_right', 'mouth_left', 'mouth_right', + 'lips_roll_in', 'snarl', 'tongue_center' +] + +# Per-vertex material ids (priority ordered, later wins). +MATERIALS = ['skin', 'teeth', 'gums', 'tongue', 'scleras', 'irises', 'pupils'] + +DTYPE_NAMES = { + np.dtype(np.float32): 'float32', + np.dtype(np.int8): 'int8', + np.dtype(np.uint8): 'uint8', + np.dtype(np.uint16): 'uint16', + np.dtype(np.int32): 'int32', +} + + +class ContainerWriter: + """Accumulates named binary sections and writes a GNMW container file.""" + + def __init__(self): + self.sections = [] + self.blobs = [] + self.offset = 0 + + def add(self, name, array): + array = np.ascontiguousarray(array) + dtype = DTYPE_NAMES[array.dtype] + data = array.tobytes() + self.sections.append({ + 'name': name, + 'dtype': dtype, + 'shape': list(array.shape), + 'offset': self.offset, + 'byteLength': len(data), + }) + pad = (-len(data)) % 4 + self.blobs.append(data + b'\x00' * pad) + self.offset += len(data) + pad + + def write(self, path, meta): + # Section offsets are relative to the end of the header; the reader adds + # the base (12 + header length) itself. + header = {'meta': meta, 'sections': self.sections} + header_bytes = json.dumps(header, separators=(',', ':')).encode('utf-8') + pad = (-len(header_bytes)) % 4 + header_bytes += b' ' * pad + preamble = b'GNMW' + np.array([1, len(header_bytes)], + dtype=' 0) + if has_pose_correctives: + correctives_q, correctives_scales = quantize_int8(correctives) + + # ---- Validate quantization error. ---------------------------------------- + rng = np.random.default_rng(0) + coeff_id = rng.normal(size=(16, identity_dim)).astype(np.float32) + coeff_ex = rng.normal(size=(16, expression_dim)).astype(np.float32) + exact = coeff_id @ identity_basis + coeff_ex @ expression_basis + approx = ((coeff_id * identity_scales) @ identity_q.astype(np.float32) + + (coeff_ex * expression_scales) @ expression_q.astype(np.float32)) + err = np.abs(exact - approx) + print(f'Bind-pose int8 error: max {err.max() * 1000:.4f} mm, ' + f'mean {err.mean() * 1000:.5f} mm') + + if has_pose_correctives: + pose_features = rng.uniform(-0.6, 0.6, size=(16, 9 * num_joints)).astype( + np.float32) + exact_pc = pose_features @ correctives + approx_pc = (pose_features * correctives_scales) @ correctives_q.astype( + np.float32) + err_pc = np.abs(exact_pc - approx_pc) + print(f'Pose-correctives int8 error: max {err_pc.max() * 1000:.4f} mm, ' + f'mean {err_pc.mean() * 1000:.5f} mm') + else: + print('Pose-correctives regressor is all zeros; skipping export.') + + # ---- Per-vertex classification maps. ------------------------------------- + groups = d['vertex_groups'] + group_names = [str(n) for n in d['vertex_group_names']] + + component_names = [str(n) for n in d['mesh_component_names']] + component_weights = group_weight_matrix(groups, group_names, component_names) + component_id = component_weights.argmax(axis=0).astype(np.uint8) + + material_id = np.zeros(num_vertices, dtype=np.uint8) + for material_index, material in enumerate(MATERIALS): + if material == 'skin': + continue + weights = group_weight_matrix(groups, group_names, [material])[0] + material_id[weights > 1e-4] = material_index + + # The sclera/iris/pupil labels live on the interior eye layer; the visible + # surface is the transparent cornea shell (eye_exteriors). Project the + # interior colors onto the shell so eyes render with sclera + iris. + template = d['template_vertex_positions'] + exterior = np.where( + group_weight_matrix(groups, group_names, ['eye_exteriors'])[0] > 1e-4 + )[0] + interior = np.where( + group_weight_matrix(groups, group_names, ['eye_interiors'])[0] > 1e-4 + )[0] + if len(exterior) and len(interior): + d2 = ((template[exterior][:, None, :] - + template[interior][None, :, :]) ** 2).sum(-1) + material_id[exterior] = material_id[interior[d2.argmin(axis=1)]] + + region_names = [n for n in group_names if n.endswith('_region')] + region_weights = group_weight_matrix(groups, group_names, region_names) + region_max = region_weights.max(axis=0) + region_id = region_weights.argmax(axis=0).astype(np.uint8) + region_id[region_max <= 1e-4] = 255 + + # ---- Landmarks. ----------------------------------------------------------- + landmark_rows = [] + with open(landmarks_path, 'r', encoding='utf-8') as f: + for line in f: + parts = line.split() + if len(parts) >= 6: + landmark_rows.append([float(x) for x in parts[:6]]) + landmarks = np.array(landmark_rows, dtype=np.float64) + landmark_indices = landmarks[:, 0::2].astype(np.uint16) + landmark_weights = landmarks[:, 1::2].astype(np.float32) + print(f'Landmarks: {landmark_indices.shape[0]}') + + # ---- Write the model container. ------------------------------------------ + writer = ContainerWriter() + writer.add('template', d['template_vertex_positions'].astype(np.float32)) + writer.add('triangles', d['triangles'].astype(np.uint16)) + writer.add('quads', d['quads'].astype(np.uint16)) + writer.add('template_joints', + d['template_joint_positions'].astype(np.float32)) + writer.add('joint_parents', d['joint_parent_indices'].astype(np.int32)) + writer.add('skinning_weights', d['skinning_weights'].astype(np.float32)) + writer.add('joint_identity_basis', + d['joint_identity_basis'].astype(np.float32)) + writer.add('identity_basis', identity_q) + writer.add('identity_scales', identity_scales) + writer.add('expression_basis', expression_q) + writer.add('expression_scales', expression_scales) + if has_pose_correctives: + writer.add('pose_correctives', correctives_q) + writer.add('pose_correctives_scales', correctives_scales) + writer.add('component_id', component_id) + writer.add('material_id', material_id) + writer.add('region_id', region_id) + writer.add('landmark_indices', landmark_indices) + writer.add('landmark_weights', landmark_weights) + + meta = { + 'model': 'GNM Head', + 'gnmVersion': str(d['version']), + 'variant': str(d['variant']), + 'numVertices': int(num_vertices), + 'numJoints': int(num_joints), + 'identityDim': int(identity_dim), + 'expressionDim': int(expression_dim), + 'identityNames': [str(n) for n in d['identity_names']], + 'expressionNames': [str(n) for n in d['expression_names']], + 'jointNames': [str(n) for n in d['joint_names']], + 'componentNames': component_names, + 'materialNames': MATERIALS, + 'regionNames': [re.sub('_region$', '', n) for n in region_names], + 'bboxMin': [float(x) for x in template.min(axis=0)], + 'bboxMax': [float(x) for x in template.max(axis=0)], + 'hasPoseCorrectives': has_pose_correctives, + } + writer.write(os.path.join(args.out_dir, 'gnm_head_web.bin'), meta) + + # ---- Semantic sampler decoders. ------------------------------------------ + sampler_writer = ContainerWriter() + sampler_meta = {} + specs = [ + ('identity', 'identity_decoder_model.h5', GENDERS + ETHNICITIES), + ('expression', 'expression_decoder_model.h5', EXPRESSION_CLASSES), + ] + for key, filename, labels in specs: + layers = load_keras_mlp(os.path.join(sampler_dir, filename)) + layer_meta = [] + for i, layer in enumerate(layers): + writer_key_w = f'{key}_w{i}' + writer_key_b = f'{key}_b{i}' + sampler_writer.add(writer_key_w, layer['kernel']) + sampler_writer.add(writer_key_b, layer['bias']) + layer_meta.append({ + 'weights': writer_key_w, + 'bias': writer_key_b, + 'activation': layer['activation'], + 'inputDim': int(layer['kernel'].shape[0]), + 'outputDim': int(layer['kernel'].shape[1]), + }) + condition_dim = len(labels) + latent_dim = layer_meta[0]['inputDim'] - condition_dim + sampler_meta[key] = { + 'latentDim': latent_dim, + 'conditionDim': condition_dim, + 'labels': labels, + 'layers': layer_meta, + } + # Smoke test: decode a zero latent with the first class. + x = np.zeros((1, layer_meta[0]['inputDim']), dtype=np.float32) + x[0, latent_dim] = 1.0 + out = run_mlp(layers, x) + print(f'{key} decoder: latent {latent_dim} + {condition_dim} classes -> ' + f'{out.shape[1]} params (zero-latent output range ' + f'[{out.min():.3f}, {out.max():.3f}])') + + sampler_meta['identity']['genders'] = GENDERS + sampler_meta['identity']['ethnicities'] = ETHNICITIES + sampler_writer.write( + os.path.join(args.out_dir, 'gnm_samplers_web.bin'), sampler_meta) + + +if __name__ == '__main__': + main() diff --git a/gnm/shape/demos/web/tools/reference_case.json b/gnm/shape/demos/web/tools/reference_case.json new file mode 100644 index 00000000..816268ec --- /dev/null +++ b/gnm/shape/demos/web/tools/reference_case.json @@ -0,0 +1,74 @@ +{ + "sampleVertexIndices": [ + 0, + 1234, + 5678, + 9000, + 17820 + ], + "sampleVertices": [ + [ + 0.07524014151191177, + 0.132701584213881, + 0.04278368914796994 + ], + [ + 0.07066148681726514, + 0.2763427407195387, + 0.1334917336734655 + ], + [ + 0.07264009026446368, + 0.21735470224169007, + 0.07959980053316922 + ], + [ + -0.03549145619356031, + 0.1885630748765137, + 0.05491466889200118 + ], + [ + 0.009579310257967339, + 0.28682860020996653, + 0.12624630734082742 + ] + ], + "meanAbs": 0.1284833692321981, + "jointsWorld": [ + [ + 0.009713157311082021, + 0.11783295185559228, + 0.023818299447652853 + ], + [ + 0.022748622893714755, + 0.18772338394274307, + 0.05506041872352304 + ], + [ + 0.0743935550256453, + 0.28198126237605065, + 0.11654204803710386 + ], + [ + 0.02219130736267548, + 0.28373561225958843, + 0.1328420499877698 + ] + ], + "landmark0": [ + -0.039455113326315926, + 0.2715785058492075, + 0.0954536817877415 + ], + "landmark33": [ + 0.05599806164359592, + 0.2334553728585174, + 0.15864090428228947 + ], + "landmark67": [ + 0.034890903048064414, + 0.20489460443296154, + 0.15772462971053758 + ] +} \ No newline at end of file