engine-3d.js

/**
 * @module engine-3d
 * @description
 * 3D biometric pipeline for Ghostati.
 *
 * Parallel to engine.js (face-api / 128-D / euclidean distance), this module
 * implements the ImageEmbedder path:
 *   - feature embedding extracted from a canvas or video element
 *   - Cosine similarity as the distance metric (higher = more similar)
 *   - Separate localStorage DB (state.db3d) sharing IDs with the 2D DB
 *   - MATCH_THRESHOLD_3D: similarity >= threshold counts as a match
 *
 * IMPORTANT — what this is NOT:
 *   The embedder was trained for generic image similarity, not face recognition.
 *   It responds strongly to global visual context (background, lighting,
 *   colour palette) as much as to facial geometry. False positives on
 *   similar backgrounds are expected and are part of the didactic point:
 *   this is a "feature-level" engine, not a biometric one.
 *
 * This module does NOT dispatch `matchStateChanged`. All events are
 * dispatched by the orchestrator in main.js / auto-find-loop.js after
 * composing the unified payload from both engines.
 *
 */

import { state } from './state.js';
import { els } from './dom.js';
import { setLog } from './utils.js';
import { persistDb3d } from './db.js';
import { MEDIAPIPE_IMAGE_EMBEDDER_URL, MEDIAPIPE_TASKS_VISION_URL, MEDIAPIPE_WASM_URL } from './config.js';
import { t } from './i18n.js';

// ─────────────────────────────────────────────
// Model lifecycle
// ─────────────────────────────────────────────

/**
 * Load the 3D embedder and store it on state.
 * Safe to call multiple times — returns immediately if already loaded.
 *
 * Historical note: the exported name stays `loadMobileNet()` to preserve
 * the public API used by main.js, even though the implementation now loads
 * MediaPipe ImageEmbedder with MobileNetV3 Small.
 *
 * @param {object|null} embedderInstance Optional injected embedder for tests.
 * @returns {Promise<void>}
 * @see main.js – called during init after face-api models are loaded.
 */
export async function loadMobileNet(embedderInstance = null) {
   if (state.imageEmbedder) return;
   if (embedderInstance) {
      state.imageEmbedder = embedderInstance;
      return;
   }
   const { FilesetResolver, ImageEmbedder } = await import(MEDIAPIPE_TASKS_VISION_URL);
   const vision = await FilesetResolver.forVisionTasks(MEDIAPIPE_WASM_URL);
   state.imageEmbedder = await ImageEmbedder.createFromOptions(vision, {
      baseOptions: {
         modelAssetPath: MEDIAPIPE_IMAGE_EMBEDDER_URL,
         delegate: 'GPU'
      },
      runningMode: 'VIDEO'
   });
   setLog(t('image_embedder_ready_log'), 'engine-3d');
}

// ─────────────────────────────────────────────
// Core math
// ─────────────────────────────────────────────

/**
 * Cosine similarity between two numeric vectors.
 * Returns a value in [-1, 1]; 1 = identical direction.
 * Higher values mean MORE similar (opposite sign convention from L2 distance).
 *
 * @param {number[]} vecA
 * @param {number[]} vecB
 * @returns {number}
 */
export function cosineSimilarity(vecA, vecB) {
   let dot = 0, normA = 0, normB = 0;
   for (let i = 0; i < vecA.length; i++) {
      dot   += vecA[i] * vecB[i];
      normA += vecA[i] * vecA[i];
      normB += vecB[i] * vecB[i];
   }
   const denom = Math.sqrt(normA) * Math.sqrt(normB);
   return denom === 0 ? 0 : dot / denom;
}

// ─────────────────────────────────────────────
// Embedding extraction
// ─────────────────────────────────────────────

/**
 * Extract an ImageEmbedder embedding from a canvas or video element.
 *
 * NOTE: ImageEmbedder responds to global image content (background, colours,
 * texture) as much as to the face itself. This is a known characteristic
 * and is the didactic subject of the workshop.
 *
 * @param {HTMLCanvasElement|HTMLVideoElement} source
 * @returns {Promise<number[]>} Embedding vector serialisable to JSON.
 */
export async function getFaceEmbedding(source) {
   if (!state.imageEmbedder) throw new Error(t('image_embedder_not_loaded_error'));
   try {
      const result = state.imageEmbedder.embedForVideo(source, performance.now());
      const embedding = result?.embeddings?.[0]?.floatEmbedding;
      if (!embedding) {
         throw new Error(t('no_embedding_returned_error'));
      }
      return Array.from(embedding);
   } catch (err) {
      console.error(t('console_get_face_embedding_error'), err);
      throw new Error(t('embedding_extraction_error', { message: err.message }));
   }
}

// ─────────────────────────────────────────────
// DB operations
// ─────────────────────────────────────────────

/**
 * Scan the 3D DB for the best cosine-similarity match against `embedding`.
 *
 * @param {number[]} embedding  Query vector.
 * @returns {{ liveMaxSim: number|null, liveMaxId: number|null }}
 */
export function seekFaceInDb3d(embedding) {
   if (!state.db3d || state.db3d.faces.length === 0) {
      return { liveMaxSim: null, liveMaxId: null };
   }
   let maxSim = -Infinity, maxId = null;
   for (const rec of state.db3d.faces) {
      const sim = cosineSimilarity(embedding, rec.descriptor3d);
      if (sim > maxSim) { maxSim = sim; maxId = rec.id; }
   }
   return { liveMaxSim: maxSim, liveMaxId: maxId };
}

/**
 * Save an ImageEmbedder embedding to state.db3d under the given `id`.
 * The `id` is assigned by engine.js (saveFace) and passed here so both
 * DBs stay in sync without a shared counter.
 *
 * @param {number} id  ID already assigned by the 2D engine's saveFace.
 * @returns {Promise<{ id: number, liveInfo3d: object }|null>}
 *   Returns null if model not ready or DB not initialised.
 */
export async function saveFace3d(id) {
   if (!state.imageEmbedder) {
      setLog(t('image_embedder_not_ready_save_log'), 'engine-3d');
      return null;
   }
   if (!state.db3d) {
      setLog(t('db_3d_not_initialized_log'), 'engine-3d');
      return null;
   }
   let embedding;
   try {
      embedding = await getFaceEmbedding(els.video);
   } catch (err) {
      setLog(t('embedding_extraction_log', { message: err.message }), 'engine-3d');
      return null;
   }
   state.db3d.faces.push({
      id,
      descriptor3d: embedding,
      savedAt: new Date().toISOString(),
   });
   persistDb3d();
   setLog(t('image_embedder_saved_log', { id }), 'engine-3d');
   // After saving, seek to get liveInfo (best match is the record we just saved: sim ≈ 1)
   const liveInfo3d = seekFaceInDb3d(embedding);
   return { id, liveInfo3d };
}

// ─────────────────────────────────────────────
// Compositing (3D efficacy test)
// ─────────────────────────────────────────────

/**
 * Helper: is any 2D or 3D plugin currently active?
 * Mirrors hasActivePlugin() from engine.js without importing it.
 * @returns {boolean}
 */
function hasActivePlugin3d() {
   const G = window.gstmxx;
   const a2d = typeof G?.getActiveEffect  === 'function' && G.getActiveEffect();
   const a3d = typeof G?.getActiveEffect3d === 'function' && G.getActiveEffect3d();
   return !!(a2d || a3d);
}

/**
 * Build a composited canvas and extract an ImageEmbedder embedding from it.
 *
 * The base of the composite is, in order of preference:
 *   1. `baseCanvas` — the 2D-composited frame from engine.js's
 *      `compositeAndDetect` (video + active 2D Ghostyle already baked in). This
 *      is what makes a 2D overlay such as the Face Brush actually move the 3D
 *      similarity score: the embedder must see the *painted* pixels, not the
 *      bare video. Without it the ImageEmbedder embeds the un-obfuscated frame
 *      and reports a match no matter how much the user paints.
 *   2. the raw video frame, when no 2D composite was supplied (e.g. only a
 *      3D/UV plugin is active).
 *
 * On top of that base it dispatches `beforeEfficacyComposite3d` so 3D plugins
 * can draw before the embedding is extracted. Plugins receive:
 *   `{ canvas, ctx, landmarks3d: state.lastLandmarks3d }`
 *
 * The base is copied into a fresh canvas (never mutated in place), so passing
 * the 2D path's canvas here is safe even though 3D plugins may draw on top.
 *
 * @param {HTMLCanvasElement|null} [baseCanvas=null] Pre-composited 2D frame to embed.
 * @returns {Promise<{ canvas: HTMLCanvasElement, embedding: number[] }|null>}
 *   Returns null if model not ready.
 */
export async function compositeAndDetect3d(baseCanvas = null) {
   if (!state.imageEmbedder) return null;

   const canvas = document.createElement('canvas');
   canvas.width  = els.overlay.width;
   canvas.height = els.overlay.height;
   const ctx = canvas.getContext('2d');

   // Prefer the 2D-composited frame (video + 2D Ghostyle) so 2D makeup is
   // reflected in the embedding; otherwise fall back to the raw video frame.
   const source = baseCanvas || els.video;
   ctx.drawImage(source, 0, 0, canvas.width, canvas.height);

   // Allow 3D ghostyle plugins to draw onto this canvas before embedding.
   state.gstmxxEvents.dispatchEvent(new CustomEvent('beforeEfficacyComposite3d', {
      detail: { canvas, ctx, landmarks3d: state.lastLandmarks3d }
   }));

   const embedding = await getFaceEmbedding(canvas);
   return { canvas, embedding };
}

// ─────────────────────────────────────────────
// Find pipeline
// ─────────────────────────────────────────────

/**
 * Run the full 3D find pipeline:
 *   1. Extract embedding from the current video frame.
 *   2. Find best cosine-similarity match in state.db3d.
 *   3. If a plugin is active, also extract composite embedding.
 *   4. Return raw metrics for the orchestrator to compose into the
 *      unified `matchStateChanged` payload — does NOT dispatch any event.
 *
 * Returns null if:
 *   - ImageEmbedder not loaded yet
 *   - state.db3d is empty (3D DB has no entries)
 *
 * Returns `{ liveInfo3d, composite3d }` otherwise, where composite3d is
 * null when no plugin is active.
 *
 * @param {HTMLCanvasElement|null} [baseCanvas=null] 2D-composited frame (video +
 *   active 2D Ghostyle) to embed for the composite comparison. Passed straight
 *   through to `compositeAndDetect3d`; when omitted the composite falls back to
 *   the raw video frame.
 * @returns {Promise<{ liveInfo3d: object, composite3d: object|null }|null>}
 */
export async function findFace3d(baseCanvas = null) {
   if (!state.imageEmbedder) {
      setLog(t('image_embedder_not_ready_compare_log'), 'engine-3d');
      return null;
   }
   if (!state.db3d || state.db3d.faces.length === 0) {
      return { liveInfo3d: { liveMaxSim: null, liveMaxId: null }, composite3d: null };
   }

   let liveEmbedding;
   try {
      liveEmbedding = await getFaceEmbedding(els.video);
   } catch (err) {
      setLog(t('live_embedding_extraction_log', { message: err.message }), 'engine-3d');
      return null;
   }

   const liveInfo3d = seekFaceInDb3d(liveEmbedding);
   const composite3d = hasActivePlugin3d() ? await compositeAndDetect3d(baseCanvas) : null;

   return { liveInfo3d, composite3d };
}

// ─────────────────────────────────────────────
// Match decision
// ─────────────────────────────────────────────

/**
 * Decide match state for the 3D (cosine-similarity) engine.
 *
 * Cosine similarity convention: HIGHER = MORE SIMILAR.
 * threshold: similarity >= MATCH_THRESHOLD_3D → matched.
 *
 * When a ghostyle is present, the obfuscated embedding is the primary signal
 * (same rationale as engine.js's decideMatchState for the 2D case).
 *
 * @param {object} params
 * @param {number|null} params.liveMaxSim   Best live similarity (highest in DB).
 * @param {number|null} params.liveMaxId    ID of best live match.
 * @param {number|null} params.obfMaxSim    Best composite similarity (null if no plugin).
 * @param {number|null} params.obfMaxId     ID of best composite match.
 * @returns {{ detectionState: string, headline: string }}
 */
export function decideMatchState3d({ liveMaxSim, liveMaxId, obfMaxSim, obfMaxId }) {
   const thr = state.MATCH_THRESHOLD_3D;

   if (obfMaxSim != null) {
      // Ghostyle active: judge on composite embedding
      if (typeof obfMaxId === 'number' && obfMaxSim >= thr) {
         return {
            detectionState: 'matched',
            headline: t('match_3d_ghostyle_matched_headline', { id: obfMaxId, similarity: obfMaxSim.toFixed(3), threshold: thr.toFixed(2) }),
         };
      }
      return {
         detectionState: 'eluded',
         headline: t('match_3d_ghostyle_eluded_headline', { similarity: (obfMaxSim ?? 0).toFixed(3), threshold: thr.toFixed(2) }),
      };
   }

   // No ghostyle: judge on live embedding
   if (liveMaxSim == null) {
      return { detectionState: 'unknown', headline: t('match_3d_empty_db_headline') };
   }
   if (liveMaxSim >= thr) {
      return {
         detectionState: 'matched',
         headline: t('match_3d_live_matched_headline', { id: liveMaxId, similarity: liveMaxSim.toFixed(3), threshold: thr.toFixed(2) }),
      };
   }
   return {
      detectionState: 'eluded',
      headline: t('match_3d_live_eluded_headline', { similarity: liveMaxSim.toFixed(3), threshold: thr.toFixed(2) }),
   };
}

/**
 * Build the `mediapipe` section of the unified `matchStateChanged` payload.
 *
 * NOTE: the field is named `mediapipe` in the payload (matching the landmark
 * source) because both tasks are powered by MediaPipe. The two models run in the
 * same MediaPipe-initiated pipeline.
 *
 * @param {{ liveInfo3d: object, composite3d: object|null }} result3d
 *   As returned by findFace3d().
 * @returns {{ detectionState, headline, liveMaxSim, liveMaxId, obfMaxSim, obfMaxId }|null}
 *   Returns null if liveInfo3d is fully empty (model not ready / no records).
 */
export function evaluateMatch3d(result3d) {
   if (!result3d) return null;
   const { liveInfo3d, composite3d } = result3d;
   if (!liveInfo3d) return null;

   const { liveMaxSim, liveMaxId } = liveInfo3d;

   let obfMaxSim = null, obfMaxId = null;
   if (composite3d?.embedding) {
      const obfInfo = seekFaceInDb3d(composite3d.embedding);
      obfMaxSim = obfInfo.liveMaxSim;
      obfMaxId  = obfInfo.liveMaxId;
   }

   const { detectionState, headline } = decideMatchState3d({ liveMaxSim, liveMaxId, obfMaxSim, obfMaxId });

   // The similarity that drove the decision (composite if present, live otherwise)
   const similarity = obfMaxSim ?? liveMaxSim;
   const matchedId  = obfMaxSim != null
      ? (obfMaxSim >= state.MATCH_THRESHOLD_3D ? obfMaxId : null)
      : (liveMaxSim != null && liveMaxSim >= state.MATCH_THRESHOLD_3D ? liveMaxId : null);

   return {
      detectionState,
      headline,
      similarity,
      matchedId,
      liveMaxSim,
      liveMaxId,
      obfMaxSim,
      obfMaxId,
   };
}