From e5c84b6f59164c822897390071ae642d725336de Mon Sep 17 00:00:00 2001 From: Alex Weinstein Date: Fri, 24 Jul 2026 17:52:27 -0700 Subject: [PATCH 1/6] feat: add Biologix public intelligence poller --- tools/biologix-public-intel/.gitignore | 7 + tools/biologix-public-intel/README.md | 144 ++ tools/biologix-public-intel/poller.py | 1566 ++++++++++++++++++++ tools/biologix-public-intel/test_poller.py | 293 ++++ 4 files changed, 2010 insertions(+) create mode 100644 tools/biologix-public-intel/.gitignore create mode 100644 tools/biologix-public-intel/README.md create mode 100644 tools/biologix-public-intel/poller.py create mode 100644 tools/biologix-public-intel/test_poller.py diff --git a/tools/biologix-public-intel/.gitignore b/tools/biologix-public-intel/.gitignore new file mode 100644 index 0000000..8b5df77 --- /dev/null +++ b/tools/biologix-public-intel/.gitignore @@ -0,0 +1,7 @@ +data/ +reports/ +*.sqlite +*.sqlite3 +*.sqlite-shm +*.sqlite-wal +__pycache__/ diff --git a/tools/biologix-public-intel/README.md b/tools/biologix-public-intel/README.md new file mode 100644 index 0000000..dfacea2 --- /dev/null +++ b/tools/biologix-public-intel/README.md @@ -0,0 +1,144 @@ +# Biologix Public Intelligence Poller + +This internal tool records low-frequency snapshots of the public Biologix +WordPress and WooCommerce storefront. It turns public catalog changes into +clearly labeled observations and inferences without logging in, probing orders, +collecting customer data, or bypassing access controls. + +## What it measures + +| Signal | Classification | What it means | +|---|---|---| +| Current stock, price, availability, SKU | Observed fact | Public value returned by the Store API | +| Popularity rank | Observed fact | Current public `orderby=popularity` position | +| Product modification time | Observed fact | Public WordPress product timestamp | +| Stock decrease or increase | Observed fact | Difference between two public snapshots | +| Probable basket | Inference | Two or more decreases whose parent-product timestamps are within five seconds | +| Displayed-price GMV signal | Estimate | Units decreased multiplied by the displayed price | +| Installed analytics tags | Observed fact | Public tags found in homepage HTML | + +The poller cannot prove payment, settled revenue, fulfillment, refunds, discounts, +customer identity, sessions, pageviews, conversion rate, or traffic sources. +WooCommerce can change stock for pending orders, cancellations, returns, restocks, +automation, and manual edits. + +## Quick start + +Requires Python 3.10 or newer and no third-party packages. + +```bash +cd tools/biologix-public-intel + +# Establish the first baseline. +python3 poller.py snapshot + +# Run another snapshot later to create change events. +python3 poller.py snapshot + +# Print the last 24 hours as Markdown. +python3 poller.py report --since 24h + +# Export snapshots, events, probable baskets, and current inventory. +python3 poller.py export --since 7d +``` + +The default database is `data/biologix-public-intel.sqlite3`. Runtime data and +exports are intentionally gitignored. + +## Continuous polling + +```bash +python3 poller.py watch --interval 900 --jitter 30 +``` + +The default interval is 15 minutes. The tool refuses intervals below five minutes. +Jitter avoids hitting the site at exactly the same second on every cycle. Stop with +`Ctrl-C`. + +For a Mac that should keep collecting after the terminal closes, use `launchd`, +`tmux`, or a supervised process and call the same `watch` command. Do not create a +high-frequency crawler. The public endpoints already expose exact timestamps, so +five-to-fifteen-minute snapshots are enough for useful direction. + +## Commands + +```text +snapshot Fetch and store one atomic snapshot +watch --interval 900 Poll continuously +report --since 24h Print a Markdown intelligence report +report --since 7d --format json Print machine-readable analysis +export --since 30d Write CSV files under reports/ +traffic-audit Show detectable public analytics tags and limits +``` + +Global options must appear before the command: + +```bash +python3 poller.py \ + --db /path/to/intel.sqlite3 \ + --base-url https://biologixlabsresearch.com \ + snapshot +``` + +## Reading the report + +Use the three evidence tiers separately: + +1. **Observed:** exact public values and changes. +2. **Inferred:** probable basket clusters supported by timestamp correlation. +3. **Unavailable:** facts that require authorized analytics or processor records. + +Never call an inventory decrease a paid sale. The useful commercial metrics are: + +- observed units down; +- displayed-price GMV signal; +- minimum correlated basket count; +- unclustered decrease candidates; +- units up, which may represent restocks, returns, cancellations, or corrections; +- rank, price, and availability movement. + +## Traffic analysis + +Public WordPress data does not expose real traffic counts. This tool inventories +public analytics tags so the installed stack is known, but tag IDs do not reveal +sessions or conversions. + +Traffic can be analyzed in three progressively stronger ways: + +1. **Public-only:** stock velocity, probable baskets, popularity-rank movement, + product/catalog updates, sitemap growth, search visibility, and third-party + traffic estimates. Third-party traffic numbers are directional. +2. **Owner-provided read-only:** GA4, Search Console, Cloudflare or Jetpack stats, + WooCommerce Analytics product exports, and affiliate-platform exports. +3. **Cash truth:** processor settlements, refunds, chargebacks, and bank deposits + reconciled to WooCommerce orders. + +The strongest authorized package is a daily sanitized export containing timestamp, +product/variation, quantity, net sales, order status, coupon amount, refund amount, +and affiliate code, with customer PII removed. + +## Data model + +- `snapshots`: one successful collection cycle and its public site signals. +- `observations`: product and variation values at that snapshot. +- `events`: stock, price, rank, availability, and catalog changes. +- `event_groups`: timestamp-correlated probable baskets. + +Every row retains its evidence level. Reports do not silently promote an inference +into a sale. + +## Safety and data quality + +- Public `GET` requests only. +- No login, credentials, cookies, order enumeration, or customer endpoints. +- Five-minute hard minimum interval. +- Short timeouts and bounded retries. +- No raw homepage storage. +- No customer or personal data. +- Spreadsheet-formula prefixes are neutralized in CSV exports. +- SQLite writes occur in one transaction per snapshot. +- Variable-product parent stock is excluded from inventory totals when child + variations already expose quantities, preventing obvious double counting. + +This is competitive and operational research, not a professional security audit +and not an accounting system. diff --git a/tools/biologix-public-intel/poller.py b/tools/biologix-public-intel/poller.py new file mode 100644 index 0000000..7210f7f --- /dev/null +++ b/tools/biologix-public-intel/poller.py @@ -0,0 +1,1566 @@ +#!/usr/bin/env python3 +"""Low-frequency public storefront intelligence for WordPress/WooCommerce.""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import random +import re +import sqlite3 +import sys +import time +from dataclasses import asdict, dataclass +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Iterable, Sequence +from urllib.error import HTTPError, URLError +from urllib.parse import parse_qsl, urlencode, urljoin, urlsplit, urlunsplit +from urllib.request import Request, urlopen + + +DEFAULT_BASE_URL = "https://biologixlabsresearch.com" +DEFAULT_INTERVAL_SECONDS = 900 +MIN_INTERVAL_SECONDS = 300 +DEFAULT_TIMEOUT_SECONDS = 20.0 +MAX_RESPONSE_BYTES = 8 * 1024 * 1024 +USER_AGENT = ( + "OVO-Public-Storefront-Research/1.0 " + "(low-frequency public GET monitoring; no customer or order data)" +) + +SCRIPT_DIR = Path(__file__).resolve().parent +DEFAULT_DB_PATH = SCRIPT_DIR / "data" / "biologix-public-intel.sqlite3" +DEFAULT_REPORT_DIR = SCRIPT_DIR / "reports" + + +SCHEMA = """ +PRAGMA journal_mode = WAL; +PRAGMA foreign_keys = ON; + +CREATE TABLE IF NOT EXISTS snapshots ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + captured_at TEXT NOT NULL UNIQUE, + base_url TEXT NOT NULL, + duration_ms INTEGER NOT NULL, + parent_count INTEGER NOT NULL, + variation_count INTEGER NOT NULL, + exact_inventory_units INTEGER NOT NULL, + displayed_inventory_value_cents INTEGER NOT NULL, + homepage_bytes INTEGER, + trackers_json TEXT NOT NULL, + response_meta_json TEXT NOT NULL, + errors_json TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS observations ( + snapshot_id INTEGER NOT NULL REFERENCES snapshots(id) ON DELETE CASCADE, + item_key TEXT NOT NULL, + record_type TEXT NOT NULL, + product_type TEXT NOT NULL, + product_id INTEGER NOT NULL, + parent_id INTEGER NOT NULL, + name TEXT NOT NULL, + variation TEXT NOT NULL, + sku TEXT NOT NULL, + price_cents INTEGER, + regular_price_cents INTEGER, + sale_price_cents INTEGER, + stock_quantity INTEGER, + stock_text TEXT NOT NULL, + in_stock INTEGER NOT NULL, + on_backorder INTEGER NOT NULL, + purchasable INTEGER NOT NULL, + track_inventory INTEGER NOT NULL, + popularity_rank INTEGER, + modified_gmt TEXT, + permalink TEXT NOT NULL, + raw_hash TEXT NOT NULL, + PRIMARY KEY (snapshot_id, item_key) +); + +CREATE INDEX IF NOT EXISTS idx_observations_item + ON observations(item_key, snapshot_id); +CREATE INDEX IF NOT EXISTS idx_observations_parent + ON observations(snapshot_id, parent_id); + +CREATE TABLE IF NOT EXISTS events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + snapshot_id INTEGER NOT NULL REFERENCES snapshots(id) ON DELETE CASCADE, + previous_snapshot_id INTEGER REFERENCES snapshots(id) ON DELETE SET NULL, + observed_at TEXT NOT NULL, + event_type TEXT NOT NULL, + item_key TEXT NOT NULL, + parent_id INTEGER NOT NULL, + name TEXT NOT NULL, + variation TEXT NOT NULL, + old_quantity INTEGER, + new_quantity INTEGER, + quantity_delta INTEGER, + old_price_cents INTEGER, + new_price_cents INTEGER, + displayed_value_cents INTEGER, + old_rank INTEGER, + new_rank INTEGER, + confidence REAL NOT NULL, + modified_gmt TEXT, + group_id TEXT, + evidence_json TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_events_observed + ON events(observed_at, event_type); +CREATE INDEX IF NOT EXISTS idx_events_group + ON events(group_id); + +CREATE TABLE IF NOT EXISTS event_groups ( + group_id TEXT PRIMARY KEY, + snapshot_id INTEGER NOT NULL REFERENCES snapshots(id) ON DELETE CASCADE, + observed_at TEXT NOT NULL, + occurred_at TEXT, + group_type TEXT NOT NULL, + item_count INTEGER NOT NULL, + unit_count INTEGER NOT NULL, + displayed_value_cents INTEGER NOT NULL, + confidence REAL NOT NULL, + evidence_json TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_event_groups_observed + ON event_groups(observed_at, group_type); +""" + + +@dataclass(frozen=True) +class HttpResult: + url: str + body: bytes + status: int + headers: dict[str, str] + duration_ms: int + + +@dataclass(frozen=True) +class Observation: + item_key: str + record_type: str + product_type: str + product_id: int + parent_id: int + name: str + variation: str + sku: str + price_cents: int | None + regular_price_cents: int | None + sale_price_cents: int | None + stock_quantity: int | None + stock_text: str + in_stock: bool + on_backorder: bool + purchasable: bool + track_inventory: bool + popularity_rank: int | None + modified_gmt: str | None + permalink: str + raw_hash: str + + +def utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def isoformat_utc(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat(timespec="microseconds") + + +def parse_datetime(value: str | None) -> datetime | None: + if not value: + return None + normalized = value.replace("Z", "+00:00") + parsed = datetime.fromisoformat(normalized) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +def parse_since(value: str, now: datetime | None = None) -> str: + now = now or utc_now() + match = re.fullmatch(r"(\d+)([mhdw])", value.strip().lower()) + if match: + amount = int(match.group(1)) + unit = match.group(2) + delta = { + "m": timedelta(minutes=amount), + "h": timedelta(hours=amount), + "d": timedelta(days=amount), + "w": timedelta(weeks=amount), + }[unit] + return isoformat_utc(now - delta) + parsed = parse_datetime(value) + if parsed is None: + raise ValueError(f"Invalid --since value: {value}") + return isoformat_utc(parsed) + + +def parse_stock_quantity(stock_text: str | None, in_stock: bool) -> int | None: + text = (stock_text or "").strip() + match = re.match(r"^(-?\d+)\s+in stock\b", text, flags=re.IGNORECASE) + if match: + return int(match.group(1)) + if not in_stock and text.lower().startswith("out of stock"): + return 0 + return None + + +def cents(value: Any) -> int | None: + if value in (None, ""): + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def stable_hash(payload: dict[str, Any]) -> str: + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(encoded).hexdigest() + + +def add_query_value(url: str, key: str, value: str | int) -> str: + parts = urlsplit(url) + query = dict(parse_qsl(parts.query, keep_blank_values=True)) + query[key] = str(value) + return urlunsplit( + (parts.scheme, parts.netloc, parts.path, urlencode(query), parts.fragment) + ) + + +def fetch_url( + url: str, + *, + timeout: float = DEFAULT_TIMEOUT_SECONDS, + max_bytes: int = MAX_RESPONSE_BYTES, + attempts: int = 3, +) -> HttpResult: + last_error: Exception | None = None + for attempt in range(attempts): + started = time.monotonic() + request = Request( + url, + headers={ + "Accept": "application/json,text/html;q=0.9,*/*;q=0.1", + "User-Agent": USER_AGENT, + }, + method="GET", + ) + try: + with urlopen(request, timeout=timeout) as response: + body = response.read(max_bytes + 1) + if len(body) > max_bytes: + raise ValueError(f"Response exceeded {max_bytes} bytes: {url}") + return HttpResult( + url=response.geturl(), + body=body, + status=int(response.status), + headers={key.lower(): value for key, value in response.headers.items()}, + duration_ms=round((time.monotonic() - started) * 1000), + ) + except (HTTPError, URLError, TimeoutError, ValueError) as error: + last_error = error + if isinstance(error, HTTPError) and error.code < 500: + break + if attempt + 1 < attempts: + time.sleep(0.5 * (2**attempt)) + assert last_error is not None + raise RuntimeError(f"GET failed after {attempts} attempts: {url}: {last_error}") + + +def fetch_json_collection( + url: str, + *, + timeout: float = DEFAULT_TIMEOUT_SECONDS, + max_pages: int = 20, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + records: list[dict[str, Any]] = [] + response_meta: list[dict[str, Any]] = [] + page = 1 + total_pages = 1 + while page <= total_pages: + if page > max_pages: + raise RuntimeError(f"Collection exceeded the {max_pages}-page safety cap: {url}") + page_url = add_query_value(url, "page", page) + result = fetch_url(page_url, timeout=timeout) + decoded = json.loads(result.body) + if not isinstance(decoded, list): + raise ValueError(f"Expected a JSON list from {page_url}") + records.extend(decoded) + total_pages = int(result.headers.get("x-wp-totalpages", "1") or "1") + response_meta.append( + { + "url": page_url, + "status": result.status, + "duration_ms": result.duration_ms, + "items": len(decoded), + "total_items": result.headers.get("x-wp-total"), + "total_pages": total_pages, + "last_modified": result.headers.get("last-modified"), + } + ) + page += 1 + return records, response_meta + + +def detect_trackers(html: str) -> list[dict[str, str]]: + patterns = { + "google_tag_manager": r"\bGTM-[A-Z0-9]{5,}\b", + "google_analytics": r"\bG-[A-Z0-9]{6,}\b", + "universal_analytics": r"\bUA-\d{4,}-\d+\b", + "google_ads": r"\bAW-\d{5,}\b", + "meta_pixel": r"""fbq\(\s*['"]init['"]\s*,\s*['"](\d{5,})['"]""", + "tiktok_pixel": r"""ttq\.load\(\s*['"]([A-Z0-9]{8,})['"]""", + "microsoft_clarity": r"""clarity\.ms/tag/([a-z0-9]+)""", + "hotjar": r"""hjid\s*[:=]\s*(\d+)""", + } + found: set[tuple[str, str]] = set() + for provider, pattern in patterns.items(): + for match in re.finditer(pattern, html, flags=re.IGNORECASE): + identifier = match.group(1) if match.lastindex else match.group(0) + found.add((provider, identifier.upper())) + + generic_markers = { + "brevo": ("sibautomation.com", "sendinblue"), + "meta_pixel_present": ("connect.facebook.net/en_US/fbevents.js",), + "tiktok_pixel_present": ("analytics.tiktok.com",), + } + lowered = html.lower() + for provider, markers in generic_markers.items(): + if any(marker.lower() in lowered for marker in markers): + found.add((provider, "present")) + + return [ + {"provider": provider, "public_id": identifier} + for provider, identifier in sorted(found) + ] + + +def build_observations( + parents: Sequence[dict[str, Any]], + variations: Sequence[dict[str, Any]], + wp_products: Sequence[dict[str, Any]], +) -> list[Observation]: + modified_by_id = { + int(product["id"]): product.get("modified_gmt") + for product in wp_products + if product.get("id") is not None + } + + exact_child_parents: set[int] = set() + variation_quantities: dict[int, int | None] = {} + for variation in variations: + quantity = parse_stock_quantity( + variation.get("stock_availability", {}).get("text"), + bool(variation.get("is_in_stock")), + ) + variation_quantities[int(variation["id"])] = quantity + if quantity is not None: + exact_child_parents.add(int(variation.get("parent") or 0)) + + observations: list[Observation] = [] + + def normalize( + item: dict[str, Any], + *, + record_type: str, + rank: int | None, + quantity: int | None, + track_inventory: bool, + ) -> Observation: + product_id = int(item["id"]) + parent_id = int(item.get("parent") or product_id) + prices = item.get("prices") or {} + stock_text = str((item.get("stock_availability") or {}).get("text") or "") + selected = { + "id": product_id, + "parent": parent_id, + "name": item.get("name"), + "variation": item.get("variation"), + "sku": item.get("sku"), + "price": prices.get("price"), + "regular_price": prices.get("regular_price"), + "sale_price": prices.get("sale_price"), + "stock_quantity": quantity, + "stock_text": stock_text, + "in_stock": item.get("is_in_stock"), + "on_backorder": item.get("is_on_backorder"), + "purchasable": item.get("is_purchasable"), + "rank": rank, + "modified_gmt": modified_by_id.get(parent_id), + } + return Observation( + item_key=f"{record_type}:{product_id}", + record_type=record_type, + product_type=str(item.get("type") or ""), + product_id=product_id, + parent_id=parent_id, + name=str(item.get("name") or ""), + variation=str(item.get("variation") or ""), + sku=str(item.get("sku") or ""), + price_cents=cents(prices.get("price")), + regular_price_cents=cents(prices.get("regular_price")), + sale_price_cents=cents(prices.get("sale_price")), + stock_quantity=quantity, + stock_text=stock_text, + in_stock=bool(item.get("is_in_stock")), + on_backorder=bool(item.get("is_on_backorder")), + purchasable=bool(item.get("is_purchasable")), + track_inventory=track_inventory, + popularity_rank=rank, + modified_gmt=modified_by_id.get(parent_id), + permalink=str(item.get("permalink") or ""), + raw_hash=stable_hash(selected), + ) + + for rank, parent in enumerate(parents, start=1): + in_stock = bool(parent.get("is_in_stock")) + quantity = parse_stock_quantity( + (parent.get("stock_availability") or {}).get("text"), in_stock + ) + product_id = int(parent["id"]) + product_type = str(parent.get("type") or "") + track_parent = ( + quantity is not None + and bool(parent.get("is_purchasable")) + and ( + product_type == "simple" + or (product_type == "variable" and product_id not in exact_child_parents) + ) + ) + observations.append( + normalize( + parent, + record_type="product", + rank=rank, + quantity=quantity, + track_inventory=track_parent, + ) + ) + + for variation in variations: + quantity = variation_quantities[int(variation["id"])] + observations.append( + normalize( + variation, + record_type="variation", + rank=None, + quantity=quantity, + track_inventory=quantity is not None + and bool(variation.get("is_purchasable")), + ) + ) + + return observations + + +def connect_db(path: Path) -> sqlite3.Connection: + path.parent.mkdir(parents=True, exist_ok=True) + connection = sqlite3.connect(path) + connection.row_factory = sqlite3.Row + connection.executescript(SCHEMA) + return connection + + +def previous_snapshot_id(connection: sqlite3.Connection) -> int | None: + row = connection.execute( + "SELECT id FROM snapshots ORDER BY id DESC LIMIT 1" + ).fetchone() + return int(row["id"]) if row else None + + +def observation_rows( + connection: sqlite3.Connection, snapshot_id: int +) -> dict[str, sqlite3.Row]: + return { + row["item_key"]: row + for row in connection.execute( + "SELECT * FROM observations WHERE snapshot_id = ?", (snapshot_id,) + ) + } + + +def insert_event( + connection: sqlite3.Connection, + *, + snapshot_id: int, + previous_id: int | None, + observed_at: str, + event_type: str, + current: sqlite3.Row, + previous: sqlite3.Row | None, + quantity_delta: int | None = None, + displayed_value_cents: int | None = None, + confidence: float = 1.0, + evidence: dict[str, Any], +) -> int: + cursor = connection.execute( + """ + INSERT INTO events ( + snapshot_id, previous_snapshot_id, observed_at, event_type, + item_key, parent_id, name, variation, + old_quantity, new_quantity, quantity_delta, + old_price_cents, new_price_cents, displayed_value_cents, + old_rank, new_rank, confidence, modified_gmt, group_id, evidence_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?) + """, + ( + snapshot_id, + previous_id, + observed_at, + event_type, + current["item_key"], + current["parent_id"], + current["name"], + current["variation"], + previous["stock_quantity"] if previous else None, + current["stock_quantity"], + quantity_delta, + previous["price_cents"] if previous else None, + current["price_cents"], + displayed_value_cents, + previous["popularity_rank"] if previous else None, + current["popularity_rank"], + confidence, + current["modified_gmt"], + json.dumps(evidence, sort_keys=True), + ), + ) + return int(cursor.lastrowid) + + +def cluster_decreases( + connection: sqlite3.Connection, + *, + snapshot_id: int, + observed_at: str, + decrease_event_ids: Sequence[int], + correlation_seconds: int = 5, +) -> int: + if len(decrease_event_ids) < 2: + return 0 + placeholders = ",".join("?" for _ in decrease_event_ids) + rows = list( + connection.execute( + f"SELECT * FROM events WHERE id IN ({placeholders})", decrease_event_ids + ) + ) + if not rows: + return 0 + + previous_id = rows[0]["previous_snapshot_id"] + current_capture_row = connection.execute( + "SELECT captured_at FROM snapshots WHERE id = ?", (snapshot_id,) + ).fetchone() + previous_capture_row = ( + connection.execute( + "SELECT captured_at FROM snapshots WHERE id = ?", (previous_id,) + ).fetchone() + if previous_id is not None + else None + ) + current_capture = ( + parse_datetime(current_capture_row["captured_at"]) + if current_capture_row + else None + ) + previous_capture = ( + parse_datetime(previous_capture_row["captured_at"]) + if previous_capture_row + else None + ) + timestamp_grace = timedelta(minutes=2) + + timestamped = [ + (row, parse_datetime(row["modified_gmt"])) + for row in rows + if ( + parse_datetime(row["modified_gmt"]) is not None + and ( + previous_capture is None + or parse_datetime(row["modified_gmt"]) + >= previous_capture - timestamp_grace + ) + and ( + current_capture is None + or parse_datetime(row["modified_gmt"]) + <= current_capture + timestamp_grace + ) + ) + ] + timestamped.sort(key=lambda pair: pair[1]) + + clusters: list[list[tuple[sqlite3.Row, datetime | None]]] = [] + active: list[tuple[sqlite3.Row, datetime | None]] = [] + for pair in timestamped: + if not active: + active = [pair] + continue + assert pair[1] is not None and active[-1][1] is not None + if (pair[1] - active[-1][1]).total_seconds() <= correlation_seconds: + active.append(pair) + else: + if len(active) >= 2: + clusters.append(active) + active = [pair] + if len(active) >= 2: + clusters.append(active) + + inserted = 0 + for cluster in clusters: + event_ids = [int(pair[0]["id"]) for pair in cluster] + if len(set(event_ids)) < 2: + continue + occurred_at = min(pair[1] for pair in cluster if pair[1] is not None) + unit_count = sum(abs(int(pair[0]["quantity_delta"] or 0)) for pair in cluster) + displayed_value = sum( + int(pair[0]["displayed_value_cents"] or 0) for pair in cluster + ) + digest_input = ( + f"{snapshot_id}:{occurred_at.isoformat()}:{','.join(map(str, event_ids))}" + ) + group_id = "basket-" + hashlib.sha256(digest_input.encode()).hexdigest()[:16] + evidence = { + "classification": "probable_basket_not_confirmed_sale", + "basis": ( + f"{len(cluster)} public inventory decreases had parent-product " + f"modified timestamps within {correlation_seconds} seconds and " + "inside the snapshot interval" + ), + "event_ids": event_ids, + "limits": [ + "May be an unpaid or failed order", + "May be a manual or automated inventory adjustment", + "Displayed prices exclude discounts, tax, shipping, and refunds", + ], + } + connection.execute( + """ + INSERT OR IGNORE INTO event_groups ( + group_id, snapshot_id, observed_at, occurred_at, group_type, + item_count, unit_count, displayed_value_cents, confidence, + evidence_json + ) VALUES (?, ?, ?, ?, 'probable_basket', ?, ?, ?, 0.70, ?) + """, + ( + group_id, + snapshot_id, + observed_at, + isoformat_utc(occurred_at), + len(cluster), + unit_count, + displayed_value, + json.dumps(evidence, sort_keys=True), + ), + ) + connection.execute( + f"UPDATE events SET group_id = ? WHERE id IN ({','.join('?' for _ in event_ids)})", + (group_id, *event_ids), + ) + inserted += 1 + return inserted + + +def detect_events( + connection: sqlite3.Connection, + *, + previous_id: int | None, + snapshot_id: int, + observed_at: str, +) -> dict[str, int]: + if previous_id is None: + return {"events": 0, "probable_baskets": 0} + + previous_rows = observation_rows(connection, previous_id) + current_rows = observation_rows(connection, snapshot_id) + event_count = 0 + decrease_ids: list[int] = [] + + for item_key in sorted(set(previous_rows) & set(current_rows)): + old = previous_rows[item_key] + new = current_rows[item_key] + + if ( + old["track_inventory"] + and new["track_inventory"] + and old["stock_quantity"] is not None + and new["stock_quantity"] is not None + and old["stock_quantity"] != new["stock_quantity"] + ): + delta = int(new["stock_quantity"]) - int(old["stock_quantity"]) + event_type = "inventory_decrease" if delta < 0 else "inventory_increase" + displayed_value = ( + abs(delta) * int(new["price_cents"]) + if new["price_cents"] is not None + else None + ) + event_id = insert_event( + connection, + snapshot_id=snapshot_id, + previous_id=previous_id, + observed_at=observed_at, + event_type=event_type, + current=new, + previous=old, + quantity_delta=delta, + displayed_value_cents=displayed_value, + confidence=1.0, + evidence={ + "classification": "observed_public_inventory_change", + "old_stock": old["stock_quantity"], + "new_stock": new["stock_quantity"], + "not_proof_of": [ + "payment", + "settlement", + "fulfillment", + "non-refund", + ], + }, + ) + event_count += 1 + if delta < 0: + decrease_ids.append(event_id) + + if old["price_cents"] != new["price_cents"]: + insert_event( + connection, + snapshot_id=snapshot_id, + previous_id=previous_id, + observed_at=observed_at, + event_type="price_change", + current=new, + previous=old, + confidence=1.0, + evidence={ + "classification": "observed_public_price_change", + "old_price_cents": old["price_cents"], + "new_price_cents": new["price_cents"], + }, + ) + event_count += 1 + + if ( + old["record_type"] == "product" + and old["popularity_rank"] != new["popularity_rank"] + ): + insert_event( + connection, + snapshot_id=snapshot_id, + previous_id=previous_id, + observed_at=observed_at, + event_type="popularity_rank_change", + current=new, + previous=old, + confidence=1.0, + evidence={ + "classification": "observed_public_rank_change", + "old_rank": old["popularity_rank"], + "new_rank": new["popularity_rank"], + "note": "Rank movement does not reveal the underlying sales count.", + }, + ) + event_count += 1 + + if old["in_stock"] != new["in_stock"]: + insert_event( + connection, + snapshot_id=snapshot_id, + previous_id=previous_id, + observed_at=observed_at, + event_type="availability_change", + current=new, + previous=old, + confidence=1.0, + evidence={ + "classification": "observed_public_availability_change", + "old_in_stock": bool(old["in_stock"]), + "new_in_stock": bool(new["in_stock"]), + }, + ) + event_count += 1 + + for item_key in sorted(set(current_rows) - set(previous_rows)): + new = current_rows[item_key] + insert_event( + connection, + snapshot_id=snapshot_id, + previous_id=previous_id, + observed_at=observed_at, + event_type="catalog_added", + current=new, + previous=None, + confidence=1.0, + evidence={"classification": "observed_public_catalog_addition"}, + ) + event_count += 1 + + for item_key in sorted(set(previous_rows) - set(current_rows)): + old = previous_rows[item_key] + insert_event( + connection, + snapshot_id=snapshot_id, + previous_id=previous_id, + observed_at=observed_at, + event_type="catalog_removed", + current=old, + previous=old, + confidence=1.0, + evidence={ + "classification": "observed_public_catalog_removal", + "note": "The record may have been unpublished, deleted, or temporarily hidden.", + }, + ) + event_count += 1 + + baskets = cluster_decreases( + connection, + snapshot_id=snapshot_id, + observed_at=observed_at, + decrease_event_ids=decrease_ids, + ) + return {"events": event_count, "probable_baskets": baskets} + + +def collect_snapshot( + *, + connection: sqlite3.Connection, + base_url: str, + timeout: float = DEFAULT_TIMEOUT_SECONDS, +) -> dict[str, Any]: + started = time.monotonic() + base_url = base_url.rstrip("/") + "/" + parent_url = urljoin( + base_url, + "wp-json/wc/store/v1/products?per_page=100&orderby=popularity&order=desc", + ) + variation_url = urljoin( + base_url, + "wp-json/wc/store/v1/products?per_page=100&type=variation", + ) + wp_products_url = urljoin( + base_url, + ( + "wp-json/wp/v2/product?per_page=100" + "&_fields=id,modified,modified_gmt,title" + ), + ) + + parents, parent_meta = fetch_json_collection(parent_url, timeout=timeout) + variations, variation_meta = fetch_json_collection(variation_url, timeout=timeout) + wp_products, wp_meta = fetch_json_collection(wp_products_url, timeout=timeout) + homepage = fetch_url(base_url, timeout=timeout, max_bytes=3 * 1024 * 1024) + trackers = detect_trackers(homepage.body.decode("utf-8", errors="replace")) + + observations = build_observations(parents, variations, wp_products) + exact_inventory_units = sum( + observation.stock_quantity or 0 + for observation in observations + if observation.track_inventory and observation.stock_quantity is not None + ) + displayed_inventory_value_cents = sum( + (observation.stock_quantity or 0) * (observation.price_cents or 0) + for observation in observations + if observation.track_inventory and observation.stock_quantity is not None + ) + captured_at = isoformat_utc(utc_now()) + previous_id = previous_snapshot_id(connection) + + response_meta = { + "parents": parent_meta, + "variations": variation_meta, + "wp_products": wp_meta, + "homepage": { + "url": homepage.url, + "status": homepage.status, + "duration_ms": homepage.duration_ms, + "server": homepage.headers.get("server"), + "platform": homepage.headers.get("platform"), + "panel": homepage.headers.get("panel"), + "cache": homepage.headers.get("x-litespeed-cache"), + }, + } + duration_ms = round((time.monotonic() - started) * 1000) + + with connection: + cursor = connection.execute( + """ + INSERT INTO snapshots ( + captured_at, base_url, duration_ms, parent_count, variation_count, + exact_inventory_units, displayed_inventory_value_cents, + homepage_bytes, trackers_json, response_meta_json, errors_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, '[]') + """, + ( + captured_at, + base_url, + duration_ms, + len(parents), + len(variations), + exact_inventory_units, + displayed_inventory_value_cents, + len(homepage.body), + json.dumps(trackers, sort_keys=True), + json.dumps(response_meta, sort_keys=True), + ), + ) + snapshot_id = int(cursor.lastrowid) + connection.executemany( + """ + INSERT INTO observations ( + snapshot_id, item_key, record_type, product_type, product_id, + parent_id, name, variation, sku, price_cents, + regular_price_cents, sale_price_cents, stock_quantity, stock_text, + in_stock, on_backorder, purchasable, track_inventory, + popularity_rank, modified_gmt, permalink, raw_hash + ) VALUES ( + :snapshot_id, :item_key, :record_type, :product_type, :product_id, + :parent_id, :name, :variation, :sku, :price_cents, + :regular_price_cents, :sale_price_cents, :stock_quantity, :stock_text, + :in_stock, :on_backorder, :purchasable, :track_inventory, + :popularity_rank, :modified_gmt, :permalink, :raw_hash + ) + """, + [ + { + "snapshot_id": snapshot_id, + **asdict(observation), + "in_stock": int(observation.in_stock), + "on_backorder": int(observation.on_backorder), + "purchasable": int(observation.purchasable), + "track_inventory": int(observation.track_inventory), + } + for observation in observations + ], + ) + event_summary = detect_events( + connection, + previous_id=previous_id, + snapshot_id=snapshot_id, + observed_at=captured_at, + ) + + return { + "snapshot_id": snapshot_id, + "captured_at": captured_at, + "duration_ms": duration_ms, + "parent_count": len(parents), + "variation_count": len(variations), + "exact_inventory_units": exact_inventory_units, + "displayed_inventory_value_cents": displayed_inventory_value_cents, + "trackers": trackers, + **event_summary, + } + + +def latest_snapshot(connection: sqlite3.Connection) -> sqlite3.Row | None: + return connection.execute( + "SELECT * FROM snapshots ORDER BY id DESC LIMIT 1" + ).fetchone() + + +def product_activity( + connection: sqlite3.Connection, *, snapshot_id: int, since: str +) -> dict[str, Any]: + since_datetime = parse_datetime(since) + rows = [ + dict(row) + for row in connection.execute( + """ + SELECT product_id, name, modified_gmt + FROM observations + WHERE snapshot_id = ? + AND record_type = 'product' + AND modified_gmt IS NOT NULL + ORDER BY modified_gmt + """, + (snapshot_id,), + ) + if ( + parse_datetime(row["modified_gmt"]) is not None + and ( + since_datetime is None + or parse_datetime(row["modified_gmt"]) >= since_datetime + ) + ) + ] + timestamped = [ + (row, parse_datetime(row["modified_gmt"])) + for row in rows + if parse_datetime(row["modified_gmt"]) is not None + ] + clusters: list[list[tuple[dict[str, Any], datetime | None]]] = [] + active: list[tuple[dict[str, Any], datetime | None]] = [] + for pair in timestamped: + if not active: + active = [pair] + continue + assert pair[1] is not None and active[-1][1] is not None + if (pair[1] - active[-1][1]).total_seconds() <= 5: + active.append(pair) + else: + clusters.append(active) + active = [pair] + if active: + clusters.append(active) + + return { + "products_modified": len(rows), + "timestamp_clusters": len(clusters), + "classification": "observed_product_activity_not_sales", + "clusters": [ + { + "occurred_at": isoformat_utc( + min(pair[1] for pair in cluster if pair[1] is not None) + ), + "product_count": len(cluster), + "products": [pair[0]["name"] for pair in cluster], + } + for cluster in reversed(clusters) + ], + } + + +def report_data( + connection: sqlite3.Connection, *, since: str +) -> dict[str, Any]: + latest = latest_snapshot(connection) + if latest is None: + raise RuntimeError("No snapshots exist. Run `snapshot` first.") + + latest_id = int(latest["id"]) + movement = connection.execute( + """ + SELECT + COALESCE(SUM(CASE WHEN event_type = 'inventory_decrease' + THEN ABS(quantity_delta) ELSE 0 END), 0) AS units_down, + COALESCE(SUM(CASE WHEN event_type = 'inventory_decrease' + THEN displayed_value_cents ELSE 0 END), 0) AS displayed_gmv_cents, + COALESCE(SUM(CASE WHEN event_type = 'inventory_increase' + THEN quantity_delta ELSE 0 END), 0) AS units_up, + COALESCE(SUM(CASE WHEN event_type = 'price_change' THEN 1 ELSE 0 END), 0) + AS price_changes, + COALESCE(SUM(CASE WHEN event_type = 'popularity_rank_change' + THEN 1 ELSE 0 END), 0) AS rank_changes, + COALESCE(SUM(CASE WHEN event_type = 'availability_change' + THEN 1 ELSE 0 END), 0) AS availability_changes + FROM events + WHERE observed_at >= ? + """, + (since,), + ).fetchone() + basket_summary = connection.execute( + """ + SELECT + COUNT(*) AS basket_count, + COALESCE(SUM(unit_count), 0) AS units, + COALESCE(SUM(displayed_value_cents), 0) AS displayed_value_cents + FROM event_groups + WHERE observed_at >= ? AND group_type = 'probable_basket' + """, + (since,), + ).fetchone() + grouped_decrease_count = connection.execute( + """ + SELECT COUNT(*) AS count + FROM events + WHERE observed_at >= ? + AND event_type = 'inventory_decrease' + AND group_id IS NOT NULL + """, + (since,), + ).fetchone()["count"] + total_decrease_count = connection.execute( + """ + SELECT COUNT(*) AS count + FROM events + WHERE observed_at >= ? AND event_type = 'inventory_decrease' + """, + (since,), + ).fetchone()["count"] + + top_products = [ + dict(row) + for row in connection.execute( + """ + SELECT popularity_rank, product_id, name, price_cents, stock_text, + in_stock, permalink + FROM observations + WHERE snapshot_id = ? + AND record_type = 'product' + AND popularity_rank IS NOT NULL + ORDER BY popularity_rank + LIMIT 12 + """, + (latest_id,), + ) + ] + recent_events = [ + dict(row) + for row in connection.execute( + """ + SELECT observed_at, event_type, name, variation, quantity_delta, + old_quantity, new_quantity, old_price_cents, new_price_cents, + displayed_value_cents, old_rank, new_rank, confidence, + modified_gmt, group_id + FROM events + WHERE observed_at >= ? + ORDER BY id DESC + LIMIT 30 + """, + (since,), + ) + ] + probable_baskets = [ + { + **dict(row), + "evidence": json.loads(row["evidence_json"]), + } + for row in connection.execute( + """ + SELECT * + FROM event_groups + WHERE observed_at >= ? + ORDER BY observed_at DESC + LIMIT 30 + """, + (since,), + ) + ] + + return { + "generated_at": isoformat_utc(utc_now()), + "since": since, + "latest_snapshot": { + "id": latest_id, + "captured_at": latest["captured_at"], + "parent_count": latest["parent_count"], + "variation_count": latest["variation_count"], + "exact_inventory_units": latest["exact_inventory_units"], + "displayed_inventory_value_cents": latest[ + "displayed_inventory_value_cents" + ], + "trackers": json.loads(latest["trackers_json"]), + }, + "observed_movement": { + **dict(movement), + "inventory_decrease_events": int(total_decrease_count), + "grouped_inventory_decrease_events": int(grouped_decrease_count), + "unclustered_inventory_decrease_events": int(total_decrease_count) + - int(grouped_decrease_count), + }, + "probable_baskets": { + **dict(basket_summary), + "confidence": 0.70, + "classification": "inference_not_confirmed_sale", + "groups": probable_baskets, + }, + "top_products": top_products, + "public_product_activity": product_activity( + connection, snapshot_id=latest_id, since=since + ), + "recent_events": recent_events, + "traffic": { + "direct_counts_available": False, + "publicly_detectable": [ + "analytics tag presence", + "catalog activity", + "product modification timestamps", + "popularity-rank movement", + ], + "requires_authorized_access": [ + "sessions", + "pageviews", + "traffic sources", + "conversion rate", + "paid revenue", + "refunds and chargebacks", + ], + }, + } + + +def dollars(value: int | None) -> str: + return f"${(value or 0) / 100:,.2f}" + + +def safe_csv_value(value: Any) -> Any: + if not isinstance(value, str): + return value + if value.startswith(("=", "+", "-", "@", "\t", "\r")): + return "'" + value + return value + + +def safe_csv_row(row: sqlite3.Row) -> dict[str, Any]: + return {key: safe_csv_value(row[key]) for key in row.keys()} + + +def markdown_report(data: dict[str, Any]) -> str: + latest = data["latest_snapshot"] + movement = data["observed_movement"] + baskets = data["probable_baskets"] + activity = data["public_product_activity"] + lines = [ + "# Biologix Public Intelligence Report", + "", + f"Generated: `{data['generated_at']}`", + f"Window begins: `{data['since']}`", + f"Latest snapshot: `{latest['captured_at']}`", + "", + "## Current public inventory", + "", + f"- Parent products: **{latest['parent_count']}**", + f"- Variations: **{latest['variation_count']}**", + f"- Exact countable units: **{latest['exact_inventory_units']:,}**", + ( + "- Displayed-price inventory value: " + f"**{dollars(latest['displayed_inventory_value_cents'])}**" + ), + "", + "## Observed movement", + "", + f"- Units down: **{movement['units_down']:,}**", + f"- Displayed-price GMV signal: **{dollars(movement['displayed_gmv_cents'])}**", + f"- Units up: **{movement['units_up']:,}**", + ( + "- Inventory-decrease records: " + f"**{movement['inventory_decrease_events']:,}**" + ), + ( + "- Unclustered decrease candidates: " + f"**{movement['unclustered_inventory_decrease_events']:,}**" + ), + f"- Price changes: **{movement['price_changes']:,}**", + f"- Popularity-rank changes: **{movement['rank_changes']:,}**", + f"- Availability changes: **{movement['availability_changes']:,}**", + "", + "## Probable baskets", + "", + f"- Correlated groups: **{baskets['basket_count']:,}**", + f"- Units in correlated groups: **{baskets['units']:,}**", + ( + "- Displayed-price value in correlated groups: " + f"**{dollars(baskets['displayed_value_cents'])}**" + ), + "- Confidence: **70% inference, not a confirmed sale**", + "", + "## Public product activity", + "", + ( + "- Products whose latest public modification falls in this window: " + f"**{activity['products_modified']:,}**" + ), + ( + "- Distinct five-second timestamp clusters: " + f"**{activity['timestamp_clusters']:,}**" + ), + ( + "- Classification: **observed product activity, not sales or traffic**" + ), + ] + for cluster in activity["clusters"][:10]: + lines.append( + f"- `{cluster['occurred_at']}`: {', '.join(cluster['products'])}" + ) + + lines.extend( + [ + "", + "## Current public popularity order", + "", + "| Rank | Product | Displayed price | Stock text |", + "|---:|---|---:|---|", + ] + ) + for product in data["top_products"]: + lines.append( + f"| {product['popularity_rank']} | {product['name']} | " + f"{dollars(product['price_cents'])} | {product['stock_text'] or '—'} |" + ) + + lines.extend( + [ + "", + "## Traffic visibility", + "", + ( + "- Direct sessions, pageviews, sources, and conversion rate are " + "**not publicly available**." + ), + ( + "- Public analytics tags detected: " + f"`{json.dumps(latest['trackers'], sort_keys=True)}`" + ), + ( + "- Exact traffic requires authorized GA4, Cloudflare, Jetpack, " + "Search Console, or equivalent access." + ), + "", + "## Recent events", + "", + ] + ) + if not data["recent_events"]: + lines.append("No changes have been observed in this window.") + else: + for event in data["recent_events"][:20]: + item = event["name"] + if event["variation"]: + item += f" ({event['variation']})" + if event["event_type"].startswith("inventory_"): + detail = ( + f"{event['old_quantity']} → {event['new_quantity']} " + f"({event['quantity_delta']:+d})" + ) + elif event["event_type"] == "price_change": + detail = ( + f"{dollars(event['old_price_cents'])} → " + f"{dollars(event['new_price_cents'])}" + ) + elif event["event_type"] == "popularity_rank_change": + detail = f"#{event['old_rank']} → #{event['new_rank']}" + else: + detail = event["event_type"].replace("_", " ") + group_note = f", `{event['group_id']}`" if event["group_id"] else "" + lines.append( + f"- `{event['observed_at']}`: **{item}**, {detail}{group_note}" + ) + + lines.extend( + [ + "", + "## Evidence boundary", + "", + ( + "Inventory movement is real public data. It is not proof of payment, " + "settlement, fulfillment, non-refund, or customer demand. Displayed " + "GMV excludes discounts, coupons, taxes, shipping, and refunds." + ), + ] + ) + return "\n".join(lines) + "\n" + + +def export_csv( + connection: sqlite3.Connection, *, since: str, output_dir: Path +) -> list[Path]: + output_dir.mkdir(parents=True, exist_ok=True) + latest = latest_snapshot(connection) + if latest is None: + raise RuntimeError("No snapshots exist. Run `snapshot` first.") + + exports: list[tuple[str, str, tuple[Any, ...]]] = [ + ( + "snapshots.csv", + "SELECT * FROM snapshots WHERE captured_at >= ? ORDER BY id", + (since,), + ), + ( + "events.csv", + "SELECT * FROM events WHERE observed_at >= ? ORDER BY id", + (since,), + ), + ( + "probable-baskets.csv", + "SELECT * FROM event_groups WHERE observed_at >= ? ORDER BY observed_at", + (since,), + ), + ( + "current-inventory.csv", + """ + SELECT item_key, record_type, product_type, product_id, parent_id, + name, variation, sku, price_cents, stock_quantity, stock_text, + in_stock, on_backorder, purchasable, track_inventory, + popularity_rank, modified_gmt, permalink + FROM observations + WHERE snapshot_id = ? + ORDER BY COALESCE(popularity_rank, 9999), name, variation + """, + (latest["id"],), + ), + ] + written: list[Path] = [] + for filename, query, params in exports: + rows = list(connection.execute(query, params)) + destination = output_dir / filename + with destination.open("w", encoding="utf-8", newline="") as handle: + if rows: + writer = csv.DictWriter(handle, fieldnames=rows[0].keys()) + writer.writeheader() + writer.writerows(safe_csv_row(row) for row in rows) + else: + handle.write("") + written.append(destination) + return written + + +def traffic_audit(connection: sqlite3.Connection) -> dict[str, Any]: + latest = latest_snapshot(connection) + if latest is None: + raise RuntimeError("No snapshots exist. Run `snapshot` first.") + response_meta = json.loads(latest["response_meta_json"]) + return { + "captured_at": latest["captured_at"], + "detected_public_trackers": json.loads(latest["trackers_json"]), + "homepage": response_meta.get("homepage", {}), + "direct_traffic_counts_available": False, + "explanation": ( + "Public tag IDs show which analytics tools may be installed. They do not " + "grant access to sessions, pageviews, sources, conversions, or revenue." + ), + "authorized_sources_needed": [ + "GA4 or Tag Manager read-only access", + "Cloudflare or Jetpack analytics", + "Google Search Console", + "WooCommerce Analytics exports", + "payment processor settlement and chargeback exports", + ], + } + + +def print_snapshot_result(result: dict[str, Any]) -> None: + print( + json.dumps( + { + **result, + "displayed_inventory_value": dollars( + result["displayed_inventory_value_cents"] + ), + }, + indent=2, + sort_keys=True, + ) + ) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Low-frequency public WordPress/WooCommerce inventory intelligence. " + "Public GET requests only." + ) + ) + parser.add_argument("--base-url", default=DEFAULT_BASE_URL) + parser.add_argument("--db", type=Path, default=DEFAULT_DB_PATH) + parser.add_argument("--timeout", type=float, default=DEFAULT_TIMEOUT_SECONDS) + subparsers = parser.add_subparsers(dest="command", required=True) + + subparsers.add_parser("snapshot", help="Collect and store one snapshot") + + watch_parser = subparsers.add_parser("watch", help="Poll continuously") + watch_parser.add_argument( + "--interval", type=int, default=DEFAULT_INTERVAL_SECONDS + ) + watch_parser.add_argument("--jitter", type=int, default=30) + + report_parser = subparsers.add_parser("report", help="Analyze collected snapshots") + report_parser.add_argument("--since", default="24h") + report_parser.add_argument( + "--format", choices=("markdown", "json"), default="markdown" + ) + + export_parser = subparsers.add_parser("export", help="Write analysis tables to CSV") + export_parser.add_argument("--since", default="30d") + export_parser.add_argument("--output-dir", type=Path, default=DEFAULT_REPORT_DIR) + + subparsers.add_parser( + "traffic-audit", help="Show detectable public analytics tags and limits" + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + connection = connect_db(args.db) + + try: + if args.command == "snapshot": + result = collect_snapshot( + connection=connection, + base_url=args.base_url, + timeout=args.timeout, + ) + print_snapshot_result(result) + return 0 + + if args.command == "watch": + if args.interval < MIN_INTERVAL_SECONDS: + parser.error( + f"--interval must be at least {MIN_INTERVAL_SECONDS} seconds" + ) + if args.jitter < 0 or args.jitter >= args.interval: + parser.error("--jitter must be non-negative and lower than --interval") + print( + f"Polling {args.base_url} every {args.interval}s " + f"(±{args.jitter}s). Stop with Ctrl-C.", + flush=True, + ) + while True: + cycle_started = time.monotonic() + try: + result = collect_snapshot( + connection=connection, + base_url=args.base_url, + timeout=args.timeout, + ) + print_snapshot_result(result) + except Exception as error: # Keep supervised polling alive. + print( + json.dumps( + { + "captured_at": isoformat_utc(utc_now()), + "error": str(error), + } + ), + file=sys.stderr, + flush=True, + ) + elapsed = time.monotonic() - cycle_started + target = args.interval + random.uniform(-args.jitter, args.jitter) + time.sleep(max(1.0, target - elapsed)) + + if args.command == "report": + since = parse_since(args.since) + data = report_data(connection, since=since) + if args.format == "json": + print(json.dumps(data, indent=2, sort_keys=True)) + else: + print(markdown_report(data), end="") + return 0 + + if args.command == "export": + since = parse_since(args.since) + files = export_csv( + connection, + since=since, + output_dir=args.output_dir, + ) + print(json.dumps({"files": [str(path) for path in files]}, indent=2)) + return 0 + + if args.command == "traffic-audit": + print(json.dumps(traffic_audit(connection), indent=2, sort_keys=True)) + return 0 + + parser.error(f"Unknown command: {args.command}") + except KeyboardInterrupt: + print("\nStopped.", file=sys.stderr) + return 130 + finally: + connection.close() + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/biologix-public-intel/test_poller.py b/tools/biologix-public-intel/test_poller.py new file mode 100644 index 0000000..736b709 --- /dev/null +++ b/tools/biologix-public-intel/test_poller.py @@ -0,0 +1,293 @@ +from __future__ import annotations + +import sqlite3 +import tempfile +import unittest +from dataclasses import replace +from datetime import datetime, timezone +from pathlib import Path + +import poller + + +def product( + product_id: int, + name: str, + *, + product_type: str = "simple", + stock: str = "10 in stock", + price: str = "1000", + parent: int = 0, + variation: str = "", + purchasable: bool = True, + in_stock: bool = True, +) -> dict: + return { + "id": product_id, + "parent": parent, + "name": name, + "type": product_type, + "variation": variation, + "sku": f"SKU-{product_id}", + "prices": { + "price": price, + "regular_price": price, + "sale_price": price, + }, + "stock_availability": {"text": stock}, + "is_in_stock": in_stock, + "is_on_backorder": False, + "is_purchasable": purchasable, + "permalink": f"https://example.test/product/{product_id}", + } + + +class PollerTests(unittest.TestCase): + def setUp(self) -> None: + self.temp_dir = tempfile.TemporaryDirectory() + self.db_path = Path(self.temp_dir.name) / "test.sqlite3" + self.connection = poller.connect_db(self.db_path) + + def tearDown(self) -> None: + self.connection.close() + self.temp_dir.cleanup() + + def insert_snapshot( + self, captured_at: str, observations: list[poller.Observation] + ) -> int: + with self.connection: + cursor = self.connection.execute( + """ + INSERT INTO snapshots ( + captured_at, base_url, duration_ms, parent_count, + variation_count, exact_inventory_units, + displayed_inventory_value_cents, homepage_bytes, + trackers_json, response_meta_json, errors_json + ) VALUES (?, 'https://example.test/', 1, 0, 0, 0, 0, 0, '[]', '{}', '[]') + """, + (captured_at,), + ) + snapshot_id = int(cursor.lastrowid) + for observation in observations: + values = poller.asdict(observation) + self.connection.execute( + """ + INSERT INTO observations ( + snapshot_id, item_key, record_type, product_type, + product_id, parent_id, name, variation, sku, + price_cents, regular_price_cents, sale_price_cents, + stock_quantity, stock_text, in_stock, on_backorder, + purchasable, track_inventory, popularity_rank, + modified_gmt, permalink, raw_hash + ) VALUES ( + :snapshot_id, :item_key, :record_type, :product_type, + :product_id, :parent_id, :name, :variation, :sku, + :price_cents, :regular_price_cents, :sale_price_cents, + :stock_quantity, :stock_text, :in_stock, :on_backorder, + :purchasable, :track_inventory, :popularity_rank, + :modified_gmt, :permalink, :raw_hash + ) + """, + { + "snapshot_id": snapshot_id, + **values, + "in_stock": int(observation.in_stock), + "on_backorder": int(observation.on_backorder), + "purchasable": int(observation.purchasable), + "track_inventory": int(observation.track_inventory), + }, + ) + return snapshot_id + + def test_stock_parsing(self) -> None: + self.assertEqual(poller.parse_stock_quantity("24 in stock", True), 24) + self.assertEqual( + poller.parse_stock_quantity("24 in stock (can be backordered)", True), 24 + ) + self.assertEqual(poller.parse_stock_quantity("Out of stock", False), 0) + self.assertIsNone(poller.parse_stock_quantity("", True)) + + def test_observation_model_avoids_variable_parent_double_count(self) -> None: + parents = [ + product(1, "Simple", stock="4 in stock"), + product(2, "Variable", product_type="variable", stock="20 in stock"), + ] + variations = [ + product( + 21, + "Variable", + product_type="variation", + stock="7 in stock", + parent=2, + variation="Amount: 10mg", + ) + ] + modified = [ + {"id": 1, "modified_gmt": "2026-07-24T20:00:00"}, + {"id": 2, "modified_gmt": "2026-07-24T20:01:00"}, + ] + observations = poller.build_observations(parents, variations, modified) + by_key = {observation.item_key: observation for observation in observations} + self.assertTrue(by_key["product:1"].track_inventory) + self.assertFalse(by_key["product:2"].track_inventory) + self.assertTrue(by_key["variation:21"].track_inventory) + self.assertEqual( + sum( + observation.stock_quantity or 0 + for observation in observations + if observation.track_inventory + ), + 11, + ) + + def test_detects_and_clusters_probable_basket(self) -> None: + source = poller.build_observations( + [ + product(1, "Alpha", stock="10 in stock", price="2500"), + product(2, "Beta", stock="8 in stock", price="5000"), + ], + [], + [ + {"id": 1, "modified_gmt": "2026-07-24T21:00:02"}, + {"id": 2, "modified_gmt": "2026-07-24T21:00:04"}, + ], + ) + changed = [ + replace( + observation, + stock_quantity=observation.stock_quantity - 1, + stock_text=f"{observation.stock_quantity - 1} in stock", + ) + for observation in source + ] + first_id = self.insert_snapshot("2026-07-24T20:55:00+00:00", source) + second_id = self.insert_snapshot("2026-07-24T21:05:00+00:00", changed) + with self.connection: + summary = poller.detect_events( + self.connection, + previous_id=first_id, + snapshot_id=second_id, + observed_at="2026-07-24T21:05:00+00:00", + ) + self.assertEqual(summary, {"events": 2, "probable_baskets": 1}) + group = self.connection.execute("SELECT * FROM event_groups").fetchone() + self.assertEqual(group["unit_count"], 2) + self.assertEqual(group["displayed_value_cents"], 7500) + self.assertAlmostEqual(group["confidence"], 0.70) + + def test_inventory_increase_is_not_a_basket(self) -> None: + source = poller.build_observations( + [product(1, "Alpha", stock="2 in stock")], + [], + [{"id": 1, "modified_gmt": "2026-07-24T21:00:00"}], + ) + changed = [replace(source[0], stock_quantity=9, stock_text="9 in stock")] + first_id = self.insert_snapshot("2026-07-24T20:55:00+00:00", source) + second_id = self.insert_snapshot("2026-07-24T21:05:00+00:00", changed) + with self.connection: + summary = poller.detect_events( + self.connection, + previous_id=first_id, + snapshot_id=second_id, + observed_at="2026-07-24T21:05:00+00:00", + ) + self.assertEqual(summary, {"events": 1, "probable_baskets": 0}) + event = self.connection.execute("SELECT * FROM events").fetchone() + self.assertEqual(event["event_type"], "inventory_increase") + self.assertEqual(event["quantity_delta"], 7) + + def test_stale_modified_times_do_not_create_probable_basket(self) -> None: + source = poller.build_observations( + [ + product(1, "Alpha", stock="10 in stock"), + product(2, "Beta", stock="10 in stock"), + ], + [], + [ + {"id": 1, "modified_gmt": "2026-07-20T21:00:02"}, + {"id": 2, "modified_gmt": "2026-07-20T21:00:04"}, + ], + ) + changed = [ + replace( + observation, + stock_quantity=observation.stock_quantity - 1, + stock_text=f"{observation.stock_quantity - 1} in stock", + ) + for observation in source + ] + first_id = self.insert_snapshot("2026-07-24T20:55:00+00:00", source) + second_id = self.insert_snapshot("2026-07-24T21:05:00+00:00", changed) + with self.connection: + summary = poller.detect_events( + self.connection, + previous_id=first_id, + snapshot_id=second_id, + observed_at="2026-07-24T21:05:00+00:00", + ) + self.assertEqual(summary, {"events": 2, "probable_baskets": 0}) + + def test_tracker_detection(self) -> None: + html = """ + + + + """ + trackers = poller.detect_trackers(html) + self.assertIn( + {"provider": "google_analytics", "public_id": "G-ABCDEF12"}, trackers + ) + self.assertIn( + {"provider": "meta_pixel", "public_id": "123456789012345"}, trackers + ) + self.assertIn({"provider": "brevo", "public_id": "present"}, trackers) + + def test_parse_since(self) -> None: + now = datetime(2026, 7, 25, 0, 0, tzinfo=timezone.utc) + self.assertEqual( + poller.parse_since("24h", now=now), + "2026-07-24T00:00:00.000000+00:00", + ) + + def test_product_activity_clusters_public_timestamps(self) -> None: + observations = poller.build_observations( + [ + product(1, "Alpha"), + product(2, "Beta"), + product(3, "Gamma"), + ], + [], + [ + {"id": 1, "modified_gmt": "2026-07-24T21:00:02"}, + {"id": 2, "modified_gmt": "2026-07-24T21:00:04"}, + {"id": 3, "modified_gmt": "2026-07-24T22:00:00"}, + ], + ) + snapshot_id = self.insert_snapshot( + "2026-07-24T23:00:00+00:00", observations + ) + activity = poller.product_activity( + self.connection, + snapshot_id=snapshot_id, + since="2026-07-24T20:00:00+00:00", + ) + self.assertEqual(activity["products_modified"], 3) + self.assertEqual(activity["timestamp_clusters"], 2) + self.assertEqual(activity["clusters"][1]["products"], ["Alpha", "Beta"]) + + def test_cli_rejects_sub_five_minute_interval(self) -> None: + parser = poller.build_parser() + args = parser.parse_args(["watch", "--interval", "300"]) + self.assertEqual(args.interval, poller.MIN_INTERVAL_SECONDS) + + def test_csv_formula_values_are_neutralized(self) -> None: + self.assertEqual( + poller.safe_csv_value('=HYPERLINK("bad")'), '\'=HYPERLINK("bad")' + ) + self.assertEqual(poller.safe_csv_value("+SUM(1,1)"), "'+SUM(1,1)") + self.assertEqual(poller.safe_csv_value("Normal product"), "Normal product") + self.assertEqual(poller.safe_csv_value(42), 42) + + +if __name__ == "__main__": + unittest.main() From 0dbc312f2ca4574681200b7ede4b7d50672f3f53 Mon Sep 17 00:00:00 2001 From: Alex Weinstein Date: Fri, 24 Jul 2026 18:16:32 -0700 Subject: [PATCH 2/6] feat: deploy scheduled Biologix intelligence collector --- tools/biologix-public-intel/README.md | 32 + tools/biologix-public-intel/cloud/.gitignore | 4 + .../biologix-public-intel/cloud/package.json | 17 + .../cloud/src/biologix-intel-core.js | 561 ++++++++++++++++ .../cloud/src/biologix-intel.js | 623 ++++++++++++++++++ .../biologix-public-intel/cloud/src/worker.js | 30 + .../cloud/test/biologix-intel.test.mjs | 168 +++++ .../cloud/wrangler.jsonc | 32 + 8 files changed, 1467 insertions(+) create mode 100644 tools/biologix-public-intel/cloud/.gitignore create mode 100644 tools/biologix-public-intel/cloud/package.json create mode 100644 tools/biologix-public-intel/cloud/src/biologix-intel-core.js create mode 100644 tools/biologix-public-intel/cloud/src/biologix-intel.js create mode 100644 tools/biologix-public-intel/cloud/src/worker.js create mode 100644 tools/biologix-public-intel/cloud/test/biologix-intel.test.mjs create mode 100644 tools/biologix-public-intel/cloud/wrangler.jsonc diff --git a/tools/biologix-public-intel/README.md b/tools/biologix-public-intel/README.md index dfacea2..7b721f8 100644 --- a/tools/biologix-public-intel/README.md +++ b/tools/biologix-public-intel/README.md @@ -142,3 +142,35 @@ into a sale. This is competitive and operational research, not a professional security audit and not an accounting system. + +## Cloud deployment + +The connected Sites Worker runs the cloud collector every 15 minutes. Its +SQLite-backed Durable Object retains 120 days of snapshot summaries, event +deltas, probable-basket inferences, and public site signals. + +The cloud runtime additionally records: + +- sitemap page counts and latest public modification times; +- public WordPress route and plugin-namespace fingerprints; +- installed analytics, email, payment, cache, CDN, and storefront technology; +- whether aggregate analytics and WooCommerce report endpoints are public or + correctly require authorization; +- public DNS and origin/cache headers; +- origin response latency and response size. + +It deliberately does not collect visitors, IP addresses, cookies, customer +records, review identities, order records, cart contents, or raw homepage HTML. +Installed tag IDs do not expose the tag owner's analytics reports. + +Cloud endpoints: + +```text +GET /api/biologix-intel/health +GET /api/biologix-intel/latest +GET /api/biologix-intel/report?hours=24 +POST /api/biologix-intel/snapshot +``` + +Only the health endpoint is public. The remaining endpoints require the private +`BIOLOGIX_INTEL_TOKEN` bearer token stored in the Sites production environment. diff --git a/tools/biologix-public-intel/cloud/.gitignore b/tools/biologix-public-intel/cloud/.gitignore new file mode 100644 index 0000000..4739fe0 --- /dev/null +++ b/tools/biologix-public-intel/cloud/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +.wrangler/ +.dry-run/ +.dev.vars diff --git a/tools/biologix-public-intel/cloud/package.json b/tools/biologix-public-intel/cloud/package.json new file mode 100644 index 0000000..6a9def8 --- /dev/null +++ b/tools/biologix-public-intel/cloud/package.json @@ -0,0 +1,17 @@ +{ + "name": "biologix-public-intel-cloud", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "dev": "wrangler dev", + "test": "node --test test/biologix-intel.test.mjs", + "validate": "npm test && wrangler deploy --dry-run --outdir .dry-run" + }, + "devDependencies": { + "wrangler": "4.113.0" + }, + "engines": { + "node": ">=20.19.0" + } +} diff --git a/tools/biologix-public-intel/cloud/src/biologix-intel-core.js b/tools/biologix-public-intel/cloud/src/biologix-intel-core.js new file mode 100644 index 0000000..5535b2a --- /dev/null +++ b/tools/biologix-public-intel/cloud/src/biologix-intel-core.js @@ -0,0 +1,561 @@ +export const BIOLOGIX_BASE_URL = "https://biologixlabsresearch.com"; +export const POLL_INTERVAL_MINUTES = 15; +export const DEEP_SCAN_INTERVAL_MS = 6 * 60 * 60 * 1000; +export const RETENTION_DAYS = 120; + +const TRACKER_PATTERNS = [ + ["google_tag_manager", /\bGTM-[A-Z0-9]{5,}\b/gi], + ["google_analytics", /\bG-[A-Z0-9]{6,}\b/gi], + ["universal_analytics", /\bUA-\d{4,}-\d+\b/gi], + ["google_ads", /\bAW-\d{5,}\b/gi], + ["meta_pixel", /fbq\(\s*['"]init['"]\s*,\s*['"](\d{5,})['"]/gi], + ["tiktok_pixel", /ttq\.load\(\s*['"]([A-Z0-9]{8,})['"]/gi], + ["microsoft_clarity", /clarity\.ms\/tag\/([a-z0-9]+)/gi], + ["hotjar", /hjid\s*[:=]\s*(\d+)/gi], +]; + +const GENERIC_TRACKER_MARKERS = [ + ["brevo", ["sibautomation.com", "sendinblue"]], + ["meta_pixel_present", ["connect.facebook.net/en_us/fbevents.js"]], + ["tiktok_pixel_present", ["analytics.tiktok.com"]], + ["optinmonster", ["optinmonster.com", "omappapi.com"]], +]; + +const RELEVANT_NAMESPACE_PATTERNS = [ + /^bankful\//, + /^jetpack\//, + /^linkmoney\//, + /^omapp\//, + /^rankmath\//, + /^sendinblue-woo\//, + /^wc-admin$/, + /^wc-analytics$/, + /^wc-push-notifications$/, + /^wc-telemetry$/, + /^wc\/store/, + /^woocommerce/, + /^wpforms\//, +]; + +const PUBLIC_AGGREGATE_PROBES = [ + ["rankmath_analytics", "/wp-json/rankmath/v1/an/analyticsSummary"], + ["rankmath_dashboard", "/wp-json/rankmath/v1/an/dashboard"], + ["rankmath_keywords", "/wp-json/rankmath/v1/an/keywordsSummary"], + ["rankmath_link_stats", "/wp-json/rankmath/v1/links/links-stats"], + ["rankmath_ai_visibility", "/wp-json/rankmath/v1/ai-visibility/overview"], + ["woocommerce_sales", "/wp-json/wc/v1/reports/sales"], + ["woocommerce_top_sellers", "/wp-json/wc/v1/reports/top_sellers"], + ["woocommerce_revenue", "/wp-json/wc-analytics/reports/revenue/stats"], + ["woocommerce_orders", "/wp-json/wc-analytics/reports/orders/stats"], +]; + +export function parseStockQuantity(stockText, inStock) { + const text = String(stockText ?? "").trim(); + const match = text.match(/^(-?\d+)\s+in stock\b/i); + if (match) return Number.parseInt(match[1], 10); + if (!inStock && text.toLowerCase().startsWith("out of stock")) return 0; + return null; +} + +export function cents(value) { + if (value === null || value === undefined || value === "") return null; + const parsed = Number.parseInt(value, 10); + return Number.isFinite(parsed) ? parsed : null; +} + +export function detectTrackers(html) { + const found = new Map(); + for (const [provider, pattern] of TRACKER_PATTERNS) { + pattern.lastIndex = 0; + for (const match of html.matchAll(pattern)) { + const publicId = String(match[1] ?? match[0]).toUpperCase(); + found.set(`${provider}:${publicId}`, { provider, public_id: publicId }); + } + } + + const lowered = html.toLowerCase(); + for (const [provider, markers] of GENERIC_TRACKER_MARKERS) { + if (markers.some((marker) => lowered.includes(marker))) { + found.set(`${provider}:present`, { provider, public_id: "present" }); + } + } + + return [...found.values()].sort((a, b) => + `${a.provider}:${a.public_id}`.localeCompare(`${b.provider}:${b.public_id}`), + ); +} + +export function detectPublicTechnology(html, headers = {}) { + const lowered = html.toLowerCase(); + const plugins = new Set(); + for (const match of html.matchAll(/\/wp-content\/plugins\/([^/'"?]+)/gi)) { + const slug = match[1].toLowerCase(); + if (/^[a-z0-9][a-z0-9._-]*$/.test(slug)) plugins.add(slug); + } + + const themes = new Set(); + for (const match of html.matchAll(/\/wp-content\/themes\/([^/'"?]+)/gi)) { + themes.add(match[1].toLowerCase()); + } + + const technologies = new Set(["wordpress"]); + if (lowered.includes("woocommerce")) technologies.add("woocommerce"); + if (lowered.includes("elementor")) technologies.add("elementor"); + if (lowered.includes("rank-math")) technologies.add("rank-math"); + if (lowered.includes("litespeed")) technologies.add("litespeed"); + if (String(headers.server ?? "").toLowerCase().includes("cloudflare")) { + technologies.add("cloudflare"); + } + if (String(headers.platform ?? "").toLowerCase().includes("hostinger")) { + technologies.add("hostinger"); + } + + return { + technologies: [...technologies].sort(), + plugins: [...plugins].sort(), + themes: [...themes].sort(), + }; +} + +function normalizeObservation(item, options) { + const prices = item.prices ?? {}; + const stockText = String(item.stock_availability?.text ?? ""); + const productId = Number(item.id); + const parentId = Number(item.parent || productId); + return { + key: `${options.recordType}:${productId}`, + record_type: options.recordType, + product_type: String(item.type ?? ""), + product_id: productId, + parent_id: parentId, + name: String(item.name ?? ""), + variation: String(item.variation ?? ""), + sku: String(item.sku ?? ""), + price_cents: cents(prices.price), + regular_price_cents: cents(prices.regular_price), + sale_price_cents: cents(prices.sale_price), + stock_quantity: options.quantity, + stock_text: stockText, + in_stock: Boolean(item.is_in_stock), + on_backorder: Boolean(item.is_on_backorder), + purchasable: Boolean(item.is_purchasable), + track_inventory: Boolean(options.trackInventory), + popularity_rank: options.rank ?? null, + modified_gmt: options.modifiedById.get(parentId) ?? null, + permalink: String(item.permalink ?? ""), + }; +} + +export function buildObservations(parents, variations, wpProducts = []) { + const modifiedById = new Map( + wpProducts + .filter((product) => product?.id !== undefined) + .map((product) => [Number(product.id), product.modified_gmt ?? null]), + ); + const variationQuantities = new Map(); + const exactChildParents = new Set(); + + for (const variation of variations) { + const quantity = parseStockQuantity( + variation.stock_availability?.text, + Boolean(variation.is_in_stock), + ); + variationQuantities.set(Number(variation.id), quantity); + if (quantity !== null) exactChildParents.add(Number(variation.parent || 0)); + } + + const observations = []; + parents.forEach((parent, index) => { + const quantity = parseStockQuantity( + parent.stock_availability?.text, + Boolean(parent.is_in_stock), + ); + const productId = Number(parent.id); + const productType = String(parent.type ?? ""); + const trackInventory = + quantity !== null && + Boolean(parent.is_purchasable) && + (productType === "simple" || + (productType === "variable" && !exactChildParents.has(productId))); + observations.push( + normalizeObservation(parent, { + recordType: "product", + rank: index + 1, + quantity, + trackInventory, + modifiedById, + }), + ); + }); + + for (const variation of variations) { + const quantity = variationQuantities.get(Number(variation.id)); + observations.push( + normalizeObservation(variation, { + recordType: "variation", + rank: null, + quantity, + trackInventory: quantity !== null && Boolean(variation.is_purchasable), + modifiedById, + }), + ); + } + return observations; +} + +export function inventorySummary(observations) { + const tracked = observations.filter( + (item) => item.track_inventory && item.stock_quantity !== null, + ); + return { + exact_inventory_units: tracked.reduce( + (total, item) => total + (item.stock_quantity ?? 0), + 0, + ), + displayed_inventory_value_cents: tracked.reduce( + (total, item) => + total + (item.stock_quantity ?? 0) * (item.price_cents ?? 0), + 0, + ), + exact_quantity_records: tracked.length, + positive_stock_records: tracked.filter((item) => item.stock_quantity > 0).length, + zero_stock_records: tracked.filter((item) => item.stock_quantity === 0).length, + hidden_purchasable_quantity_records: observations.filter( + (item) => + item.record_type === "variation" && + item.purchasable && + item.stock_quantity === null, + ).length, + backorder_capable_records: observations.filter((item) => + item.stock_text.toLowerCase().includes("can be backordered"), + ).length, + max_exact_stock_quantity: tracked.reduce( + (maximum, item) => Math.max(maximum, item.stock_quantity ?? 0), + 0, + ), + }; +} + +function eventId(capturedAt, eventType, key) { + return `${capturedAt}:${eventType}:${key}`; +} + +export function diffObservations(previousByKey, current, capturedAt) { + if (!previousByKey || Object.keys(previousByKey).length === 0) return []; + const events = []; + + for (const item of current) { + const previous = previousByKey[item.key]; + if (!previous) { + events.push({ + id: eventId(capturedAt, "catalog_added", item.key), + observed_at: capturedAt, + event_type: "catalog_added", + item_key: item.key, + parent_id: item.parent_id, + name: item.name, + variation: item.variation, + sku: item.sku, + evidence_level: "observed", + }); + continue; + } + + if ( + previous.track_inventory && + item.track_inventory && + previous.stock_quantity !== null && + item.stock_quantity !== null && + previous.stock_quantity !== item.stock_quantity + ) { + const delta = item.stock_quantity - previous.stock_quantity; + events.push({ + id: eventId(capturedAt, "inventory", item.key), + observed_at: capturedAt, + event_type: delta < 0 ? "inventory_decrease" : "inventory_increase", + item_key: item.key, + parent_id: item.parent_id, + name: item.name, + variation: item.variation, + sku: item.sku, + old_value: previous.stock_quantity, + new_value: item.stock_quantity, + quantity_delta: delta, + price_cents: item.price_cents, + displayed_value_cents: + delta < 0 ? Math.abs(delta) * (item.price_cents ?? 0) : 0, + modified_gmt: item.modified_gmt, + evidence_level: "observed", + }); + } + + if (previous.price_cents !== item.price_cents) { + events.push({ + id: eventId(capturedAt, "price", item.key), + observed_at: capturedAt, + event_type: "price_change", + item_key: item.key, + parent_id: item.parent_id, + name: item.name, + variation: item.variation, + sku: item.sku, + old_value: previous.price_cents, + new_value: item.price_cents, + evidence_level: "observed", + }); + } + + if ( + item.record_type === "product" && + previous.popularity_rank !== item.popularity_rank + ) { + events.push({ + id: eventId(capturedAt, "rank", item.key), + observed_at: capturedAt, + event_type: "popularity_rank_change", + item_key: item.key, + parent_id: item.parent_id, + name: item.name, + variation: "", + old_value: previous.popularity_rank, + new_value: item.popularity_rank, + evidence_level: "observed", + }); + } + + if (previous.in_stock !== item.in_stock) { + events.push({ + id: eventId(capturedAt, "availability", item.key), + observed_at: capturedAt, + event_type: "availability_change", + item_key: item.key, + parent_id: item.parent_id, + name: item.name, + variation: item.variation, + old_value: previous.in_stock, + new_value: item.in_stock, + evidence_level: "observed", + }); + } + } + + const currentKeys = new Set(current.map((item) => item.key)); + for (const previous of Object.values(previousByKey)) { + if (!currentKeys.has(previous.key)) { + events.push({ + id: eventId(capturedAt, "catalog_removed", previous.key), + observed_at: capturedAt, + event_type: "catalog_removed", + item_key: previous.key, + parent_id: previous.parent_id, + name: previous.name, + variation: previous.variation, + sku: previous.sku, + evidence_level: "observed", + }); + } + } + return events; +} + +export function clusterProbableBaskets(events, previousCapturedAt, capturedAt) { + const windowStart = Date.parse(previousCapturedAt ?? "") || 0; + const windowEnd = Date.parse(capturedAt) + 30_000; + const candidates = events + .filter((event) => { + if (event.event_type !== "inventory_decrease" || !event.modified_gmt) { + return false; + } + const modified = Date.parse(event.modified_gmt); + return modified >= windowStart && modified <= windowEnd; + }) + .sort((a, b) => Date.parse(a.modified_gmt) - Date.parse(b.modified_gmt)); + + const rawGroups = []; + let current = []; + for (const event of candidates) { + if ( + current.length === 0 || + Date.parse(event.modified_gmt) - + Date.parse(current[current.length - 1].modified_gmt) <= + 5_000 + ) { + current.push(event); + } else { + rawGroups.push(current); + current = [event]; + } + } + if (current.length) rawGroups.push(current); + + return rawGroups + .filter((group) => group.length >= 2) + .map((group) => { + const groupId = `basket:${capturedAt}:${group + .map((event) => event.item_key) + .sort() + .join("|")}`; + for (const event of group) event.group_id = groupId; + return { + group_id: groupId, + observed_at: capturedAt, + occurred_at: group[0].modified_gmt, + item_count: group.length, + unit_count: group.reduce( + (total, event) => total + Math.abs(event.quantity_delta ?? 0), + 0, + ), + displayed_value_cents: group.reduce( + (total, event) => total + (event.displayed_value_cents ?? 0), + 0, + ), + confidence: 0.7, + classification: "probable_basket_not_confirmed_sale", + event_ids: group.map((event) => event.id), + }; + }); +} + +export function parseSitemapIndex(xml) { + const entries = []; + const sitemapBlocks = xml.match(//gi) ?? []; + for (const block of sitemapBlocks) { + const location = block.match(/([\s\S]*?)<\/loc>/i)?.[1]?.trim(); + if (!location) continue; + const lastmod = block.match(/([\s\S]*?)<\/lastmod>/i)?.[1]?.trim(); + entries.push({ location, lastmod: lastmod ?? null }); + } + return entries; +} + +export function summarizeUrlset(xml) { + const urlBlocks = xml.match(//gi) ?? []; + const lastmods = urlBlocks + .map((block) => block.match(/([\s\S]*?)<\/lastmod>/i)?.[1]?.trim()) + .filter(Boolean) + .sort(); + return { + url_count: urlBlocks.length, + latest_lastmod: lastmods.at(-1) ?? null, + }; +} + +export function relevantNamespaces(namespaces) { + return [...new Set(namespaces)] + .filter((namespace) => + RELEVANT_NAMESPACE_PATTERNS.some((pattern) => pattern.test(namespace)), + ) + .sort(); +} + +export function publicAggregateProbes() { + return [...PUBLIC_AGGREGATE_PROBES]; +} + +export function aggregateReport(dayRecords, latestState, sinceIso) { + const cutoff = Date.parse(sinceIso); + const snapshots = dayRecords + .flatMap((day) => day?.snapshots ?? []) + .filter((snapshot) => Date.parse(snapshot.captured_at) >= cutoff) + .sort((a, b) => Date.parse(a.captured_at) - Date.parse(b.captured_at)); + const events = dayRecords + .flatMap((day) => day?.events ?? []) + .filter((event) => Date.parse(event.observed_at) >= cutoff); + const baskets = dayRecords + .flatMap((day) => day?.baskets ?? []) + .filter((basket) => Date.parse(basket.observed_at) >= cutoff); + + const decreases = events.filter( + (event) => event.event_type === "inventory_decrease", + ); + const increases = events.filter( + (event) => event.event_type === "inventory_increase", + ); + const latest = latestState?.latest_snapshot ?? null; + return { + generated_at: new Date().toISOString(), + window: { + since: sinceIso, + first_snapshot_at: snapshots[0]?.captured_at ?? null, + last_snapshot_at: snapshots.at(-1)?.captured_at ?? null, + snapshot_count: snapshots.length, + }, + current: latest, + movement: { + observed_units_down: decreases.reduce( + (total, event) => total + Math.abs(event.quantity_delta ?? 0), + 0, + ), + observed_units_up: increases.reduce( + (total, event) => total + Math.abs(event.quantity_delta ?? 0), + 0, + ), + displayed_price_gmv_signal_cents: decreases.reduce( + (total, event) => total + (event.displayed_value_cents ?? 0), + 0, + ), + inventory_decrease_records: decreases.length, + inventory_increase_records: increases.length, + price_changes: events.filter((event) => event.event_type === "price_change") + .length, + popularity_rank_changes: events.filter( + (event) => event.event_type === "popularity_rank_change", + ).length, + availability_changes: events.filter( + (event) => event.event_type === "availability_change", + ).length, + }, + probable_baskets: { + count: baskets.length, + units: baskets.reduce((total, basket) => total + basket.unit_count, 0), + displayed_value_cents: baskets.reduce( + (total, basket) => total + basket.displayed_value_cents, + 0, + ), + classification: "inference_not_confirmed_sale", + confidence: 0.7, + recent: baskets.slice(-50).reverse(), + }, + recent_inventory_events: [...decreases, ...increases] + .sort((a, b) => Date.parse(b.observed_at) - Date.parse(a.observed_at)) + .slice(0, 100), + public_site_signals: latestState?.public_site_signals ?? null, + traffic: { + direct_visitor_counts_available: false, + observed_public_signals: [ + "storefront inventory movement", + "public popularity-order movement", + "public product modification timestamps", + "sitemap growth and modification times", + "public analytics-tag and technology presence", + "origin response timing and cache headers", + ], + unavailable_without_authorized_or_paid_data: [ + "visitors", + "sessions", + "pageviews", + "traffic sources", + "conversion rate", + "paid and settled sales", + "refunds and chargebacks", + ], + }, + evidence_boundary: { + observed: + "Public GET responses and changes between scheduled snapshots.", + inferred: + "Probable baskets use correlated inventory decreases and public modification timestamps.", + unavailable: + "Inventory movement cannot prove payment, settlement, fulfillment, customer identity, or traffic.", + }, + }; +} + +export function compactObservationMap(observations) { + return Object.fromEntries(observations.map((item) => [item.key, item])); +} + +export function safeDurationSince(value, now = Date.now()) { + const timestamp = Date.parse(value ?? ""); + return Number.isFinite(timestamp) ? Math.max(0, now - timestamp) : null; +} diff --git a/tools/biologix-public-intel/cloud/src/biologix-intel.js b/tools/biologix-public-intel/cloud/src/biologix-intel.js new file mode 100644 index 0000000..dd52506 --- /dev/null +++ b/tools/biologix-public-intel/cloud/src/biologix-intel.js @@ -0,0 +1,623 @@ +import { DurableObject } from "cloudflare:workers"; + +import { + BIOLOGIX_BASE_URL, + DEEP_SCAN_INTERVAL_MS, + POLL_INTERVAL_MINUTES, + RETENTION_DAYS, + aggregateReport, + buildObservations, + clusterProbableBaskets, + compactObservationMap, + detectPublicTechnology, + detectTrackers, + diffObservations, + inventorySummary, + parseSitemapIndex, + publicAggregateProbes, + relevantNamespaces, + safeDurationSince, + summarizeUrlset, +} from "./biologix-intel-core.js"; + +const STORE_NAME = "biologix-production"; +const REQUEST_TIMEOUT_MS = 18_000; +const MAX_TEXT_BYTES = 2 * 1024 * 1024; +const MAX_DAILY_EVENTS = 300; +const MAX_SITEMAPS = 20; +const API_PREFIX = "/api/biologix-intel"; + +function jsonResponse(payload, status = 200, extraHeaders = {}) { + return new Response(JSON.stringify(payload, null, 2), { + status, + headers: { + "Cache-Control": "no-store", + "Content-Type": "application/json; charset=utf-8", + "Referrer-Policy": "no-referrer", + "X-Content-Type-Options": "nosniff", + "X-Robots-Tag": "noindex, nofollow, noarchive", + ...extraHeaders, + }, + }); +} + +function utcDateKey(iso) { + return iso.slice(0, 10); +} + +function clampReportHours(value) { + const parsed = Number.parseInt(value ?? "24", 10); + if (!Number.isFinite(parsed)) return 24; + return Math.min(RETENTION_DAYS * 24, Math.max(1, parsed)); +} + +function publicHeaders(response) { + const headerNames = [ + "cache-control", + "cf-cache-status", + "last-modified", + "platform", + "server", + "x-litespeed-cache", + "x-powered-by", + "x-turbo-charged-by", + ]; + return Object.fromEntries( + headerNames + .map((name) => [name, response.headers.get(name)]) + .filter(([, value]) => value !== null), + ); +} + +async function fetchBounded(url, options = {}) { + const started = Date.now(); + const response = await fetch(url, { + method: "GET", + redirect: "follow", + headers: { + Accept: options.accept ?? "application/json,text/html;q=0.9,*/*;q=0.1", + "User-Agent": + "OVO-Public-Intelligence/1.0 (+low-frequency aggregate research)", + }, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + const buffer = await response.arrayBuffer(); + if (buffer.byteLength > (options.maxBytes ?? MAX_TEXT_BYTES)) { + throw new Error(`Response exceeded byte limit: ${url}`); + } + return { + response, + text: new TextDecoder().decode(buffer), + bytes: buffer.byteLength, + duration_ms: Date.now() - started, + }; +} + +function withQuery(url, values) { + const parsed = new URL(url); + for (const [key, value] of Object.entries(values)) { + parsed.searchParams.set(key, String(value)); + } + return parsed.toString(); +} + +async function fetchJsonCollection(url) { + const first = await fetchBounded(withQuery(url, { page: 1 }), { + accept: "application/json", + maxBytes: 8 * 1024 * 1024, + }); + if (!first.response.ok) { + throw new Error(`GET ${url} returned ${first.response.status}`); + } + const firstRecords = JSON.parse(first.text); + if (!Array.isArray(firstRecords)) { + throw new Error(`Expected JSON list from ${url}`); + } + const totalPages = Math.min( + 20, + Number.parseInt(first.response.headers.get("x-wp-totalpages") ?? "1", 10), + ); + const remaining = + totalPages > 1 + ? await Promise.all( + Array.from({ length: totalPages - 1 }, async (_, index) => { + const page = index + 2; + const result = await fetchBounded(withQuery(url, { page }), { + accept: "application/json", + maxBytes: 8 * 1024 * 1024, + }); + if (!result.response.ok) { + throw new Error(`GET ${url} page ${page} returned ${result.response.status}`); + } + const records = JSON.parse(result.text); + if (!Array.isArray(records)) { + throw new Error(`Expected JSON list from ${url} page ${page}`); + } + return records; + }), + ) + : []; + return { + records: [...firstRecords, ...remaining.flat()], + meta: { + status: first.response.status, + total_pages: totalPages, + total_items: Number.parseInt( + first.response.headers.get("x-wp-total") ?? String(firstRecords.length), + 10, + ), + first_page_duration_ms: first.duration_ms, + }, + }; +} + +async function quickHomepageProbe() { + const result = await fetchBounded(`${BIOLOGIX_BASE_URL}/`, { + accept: "text/html", + maxBytes: 3 * 1024 * 1024, + }); + return { + status: result.response.status, + duration_ms: result.duration_ms, + bytes: result.bytes, + headers: publicHeaders(result.response), + trackers: detectTrackers(result.text), + technology: detectPublicTechnology( + result.text, + Object.fromEntries( + [...result.response.headers].map(([key, value]) => [key, value]), + ), + ), + html: result.text, + }; +} + +function publicRouteStatus(response) { + if (response.status === 401 || response.status === 403) return "authenticated"; + if (response.status === 404) return "not_found"; + if (response.ok) return "public"; + return `http_${response.status}`; +} + +async function probeAggregateEndpoint(name, path) { + try { + const result = await fetchBounded(`${BIOLOGIX_BASE_URL}${path}`, { + accept: "application/json", + maxBytes: 256 * 1024, + }); + return { + name, + path, + status_code: result.response.status, + visibility: publicRouteStatus(result.response), + }; + } catch (error) { + return { name, path, status_code: null, visibility: "error", error: error.message }; + } +} + +async function dnsQuery(type) { + const url = new URL("https://cloudflare-dns.com/dns-query"); + url.searchParams.set("name", new URL(BIOLOGIX_BASE_URL).hostname); + url.searchParams.set("type", type); + const result = await fetchBounded(url.toString(), { + accept: "application/dns-json", + maxBytes: 256 * 1024, + }); + const payload = JSON.parse(result.text); + return { + type, + status: payload.Status, + answers: (payload.Answer ?? []).map((answer) => ({ + name: answer.name, + ttl: answer.TTL, + data: answer.data, + })), + }; +} + +async function collectSitemapSignals(robotsText) { + const sitemapFromRobots = robotsText.match(/^Sitemap:\s*(\S+)/im)?.[1]; + const candidates = [ + sitemapFromRobots, + `${BIOLOGIX_BASE_URL}/sitemap_index.xml`, + `${BIOLOGIX_BASE_URL}/wp-sitemap.xml`, + ].filter(Boolean); + let indexResult = null; + for (const candidate of [...new Set(candidates)]) { + try { + const result = await fetchBounded(candidate, { + accept: "application/xml,text/xml", + maxBytes: 2 * 1024 * 1024, + }); + if (result.response.ok && result.text.includes(" { + try { + const result = await fetchBounded(entry.location, { + accept: "application/xml,text/xml", + maxBytes: 4 * 1024 * 1024, + }); + const summary = summarizeUrlset(result.text); + return { + name: new URL(entry.location).pathname.split("/").at(-1), + status: result.response.status, + ...summary, + }; + } catch (error) { + return { + name: new URL(entry.location).pathname.split("/").at(-1), + status: null, + url_count: 0, + error: error.message, + }; + } + }), + ); + return { + available: true, + index_name: new URL(indexResult.url).pathname.split("/").at(-1), + index_duration_ms: indexResult.duration_ms, + sitemap_count: childResults.length, + url_count: childResults.reduce((total, child) => total + child.url_count, 0), + sitemaps: childResults, + }; +} + +async function collectDeepSignals(homepage) { + const [robotsResult, wpRootResult, dnsResults, aggregateEndpoints] = + await Promise.all([ + fetchBounded(`${BIOLOGIX_BASE_URL}/robots.txt`, { + accept: "text/plain", + maxBytes: 256 * 1024, + }), + fetchBounded(`${BIOLOGIX_BASE_URL}/wp-json/`, { + accept: "application/json", + maxBytes: 8 * 1024 * 1024, + }), + Promise.all(["A", "AAAA", "NS", "MX"].map(dnsQuery)), + Promise.all(publicAggregateProbes().map(([name, path]) => + probeAggregateEndpoint(name, path), + )), + ]); + const wpRoot = JSON.parse(wpRootResult.text); + let jetpack = null; + try { + const result = await fetchBounded( + `${BIOLOGIX_BASE_URL}/wp-json/jetpack/v4/connection`, + { accept: "application/json", maxBytes: 128 * 1024 }, + ); + if (result.response.ok) { + const payload = JSON.parse(result.text); + jetpack = { + installed: true, + active: Boolean(payload.isActive), + registered: Boolean(payload.isRegistered), + user_connected: Boolean(payload.isUserConnected), + site_marked_public: Boolean(payload.isPublic), + }; + } + } catch { + jetpack = { installed: true, visibility: "unavailable" }; + } + + return { + captured_at: new Date().toISOString(), + robots: { + status: robotsResult.response.status, + bytes: robotsResult.bytes, + sitemap_declared: /^Sitemap:\s*(\S+)/im.test(robotsResult.text), + }, + sitemap: await collectSitemapSignals(robotsResult.text), + wordpress: { + name: String(wpRoot.name ?? ""), + description: String(wpRoot.description ?? ""), + namespace_count: Array.isArray(wpRoot.namespaces) + ? wpRoot.namespaces.length + : 0, + route_count: + wpRoot.routes && typeof wpRoot.routes === "object" + ? Object.keys(wpRoot.routes).length + : 0, + relevant_namespaces: relevantNamespaces(wpRoot.namespaces ?? []), + }, + analytics_and_sales_endpoints: aggregateEndpoints, + jetpack, + dns: dnsResults, + technology: homepage.technology, + trackers: homepage.trackers, + traffic_truth: { + visitor_counts_publicly_available: false, + reason: + "Installed tools and tags can be detected publicly, but their visitor and revenue reports require authorization.", + }, + }; +} + +async function collectPublicSnapshot(previousState, trigger) { + const started = Date.now(); + const parentUrl = `${BIOLOGIX_BASE_URL}/wp-json/wc/store/v1/products?per_page=100&orderby=popularity&order=desc`; + const variationUrl = `${BIOLOGIX_BASE_URL}/wp-json/wc/store/v1/products?per_page=100&type=variation`; + const wpProductUrl = `${BIOLOGIX_BASE_URL}/wp-json/wp/v2/product?per_page=100&_fields=id,modified_gmt`; + const [parents, variations, wpProducts, homepage] = await Promise.all([ + fetchJsonCollection(parentUrl), + fetchJsonCollection(variationUrl), + fetchJsonCollection(wpProductUrl), + quickHomepageProbe(), + ]); + + const capturedAt = new Date().toISOString(); + const observations = buildObservations( + parents.records, + variations.records, + wpProducts.records, + ); + const inventory = inventorySummary(observations); + const events = diffObservations( + previousState?.latest_observations ?? {}, + observations, + capturedAt, + ); + const baskets = clusterProbableBaskets( + events, + previousState?.latest_snapshot?.captured_at, + capturedAt, + ); + const shouldDeepScan = + safeDurationSince(previousState?.public_site_signals?.captured_at) === null || + safeDurationSince(previousState?.public_site_signals?.captured_at) >= + DEEP_SCAN_INTERVAL_MS; + let deepSignals = previousState?.public_site_signals ?? null; + let deepScanError = null; + if (shouldDeepScan) { + try { + deepSignals = await collectDeepSignals(homepage); + } catch (error) { + deepScanError = error.message; + } + } + + return { + summary: { + captured_at: capturedAt, + trigger, + duration_ms: Date.now() - started, + parent_count: parents.records.length, + variation_count: variations.records.length, + ...inventory, + homepage: { + status: homepage.status, + duration_ms: homepage.duration_ms, + bytes: homepage.bytes, + headers: homepage.headers, + trackers: homepage.trackers, + }, + event_count: events.length, + probable_basket_count: baskets.length, + deep_scan_performed: shouldDeepScan && deepScanError === null, + deep_scan_error: deepScanError, + }, + observations, + events, + baskets, + publicSiteSignals: deepSignals, + }; +} + +function newState() { + return { + version: 1, + day_keys: [], + latest_snapshot: null, + latest_observations: {}, + public_site_signals: null, + last_attempt_at: null, + last_success_at: null, + last_error_at: null, + last_error: null, + successful_runs: 0, + failed_runs: 0, + }; +} + +export class BiologixIntelStore extends DurableObject { + constructor(ctx, env) { + super(ctx, env); + this.ctx = ctx; + this.env = env; + } + + async runSnapshot(trigger = "cron") { + const state = (await this.ctx.storage.get("state")) ?? newState(); + const lastAttemptAge = safeDurationSince(state.last_attempt_at); + if (trigger === "cron" && lastAttemptAge !== null && lastAttemptAge < 4 * 60_000) { + return { skipped: true, reason: "duplicate_trigger_guard", ...this.health(state) }; + } + state.last_attempt_at = new Date().toISOString(); + await this.ctx.storage.put("state", state); + + try { + const result = await collectPublicSnapshot(state, trigger); + const dateKey = utcDateKey(result.summary.captured_at); + const storageKey = `day:${dateKey}`; + const dayRecord = (await this.ctx.storage.get(storageKey)) ?? { + date: dateKey, + snapshots: [], + events: [], + baskets: [], + events_truncated: 0, + }; + dayRecord.snapshots.push(result.summary); + dayRecord.events.push(...result.events); + dayRecord.baskets.push(...result.baskets); + if (dayRecord.events.length > MAX_DAILY_EVENTS) { + const overflow = dayRecord.events.length - MAX_DAILY_EVENTS; + dayRecord.events.splice(0, overflow); + dayRecord.events_truncated += overflow; + } + + if (!state.day_keys.includes(dateKey)) state.day_keys.push(dateKey); + state.day_keys.sort(); + const expiredKeys = state.day_keys.splice( + 0, + Math.max(0, state.day_keys.length - RETENTION_DAYS), + ); + state.latest_snapshot = result.summary; + state.latest_observations = compactObservationMap(result.observations); + state.public_site_signals = result.publicSiteSignals; + state.last_success_at = result.summary.captured_at; + state.last_error = null; + state.successful_runs += 1; + + await this.ctx.storage.put({ + state, + [storageKey]: dayRecord, + }); + if (expiredKeys.length) { + await this.ctx.storage.delete(expiredKeys.map((key) => `day:${key}`)); + } + return { + skipped: false, + snapshot: result.summary, + probable_baskets: result.baskets, + }; + } catch (error) { + state.last_error_at = new Date().toISOString(); + state.last_error = error instanceof Error ? error.message : String(error); + state.failed_runs += 1; + await this.ctx.storage.put("state", state); + throw error; + } + } + + health(state) { + const now = Date.now(); + const lastSuccessAge = safeDurationSince(state.last_success_at, now); + const healthy = + lastSuccessAge !== null && + lastSuccessAge <= (POLL_INTERVAL_MINUTES + 10) * 60_000 && + !state.last_error; + return { + status: + state.last_success_at === null ? "awaiting_first_run" : healthy ? "healthy" : "degraded", + cadence_minutes: POLL_INTERVAL_MINUTES, + last_attempt_at: state.last_attempt_at, + last_success_at: state.last_success_at, + last_success_age_seconds: + lastSuccessAge === null ? null : Math.floor(lastSuccessAge / 1000), + last_error_at: state.last_error_at, + last_error: state.last_error, + successful_runs: state.successful_runs, + failed_runs: state.failed_runs, + retention_days: RETENTION_DAYS, + collector: "public_get_only_no_customer_data", + }; + } + + async getHealth() { + const state = (await this.ctx.storage.get("state")) ?? newState(); + return this.health(state); + } + + async getLatest() { + const state = (await this.ctx.storage.get("state")) ?? newState(); + return { + health: this.health(state), + snapshot: state.latest_snapshot, + public_site_signals: state.public_site_signals, + }; + } + + async getReport(hours = 24) { + const state = (await this.ctx.storage.get("state")) ?? newState(); + const since = new Date(Date.now() - clampReportHours(hours) * 60 * 60 * 1000); + const dayKeys = state.day_keys.filter( + (key) => Date.parse(`${key}T23:59:59.999Z`) >= since.getTime(), + ); + const values = await this.ctx.storage.get( + dayKeys.map((key) => `day:${key}`), + ); + const days = dayKeys.map((key) => values.get(`day:${key}`)).filter(Boolean); + return { + health: this.health(state), + ...aggregateReport(days, state, since.toISOString()), + }; + } +} + +function getStore(env) { + if (!env.BIOLOGIX_INTEL) { + throw new Error("BIOLOGIX_INTEL Durable Object binding is unavailable"); + } + const id = env.BIOLOGIX_INTEL.idFromName(STORE_NAME); + return env.BIOLOGIX_INTEL.get(id); +} + +function isAuthorized(request, env) { + const expected = env.BIOLOGIX_INTEL_TOKEN; + if (!expected) return false; + return request.headers.get("Authorization") === `Bearer ${expected}`; +} + +export async function handleBiologixIntelRequest(request, env) { + const url = new URL(request.url); + const route = url.pathname.slice(API_PREFIX.length) || "/"; + const store = getStore(env); + + if (request.method === "GET" && route === "/health") { + return jsonResponse(await store.getHealth()); + } + + if (!isAuthorized(request, env)) { + return jsonResponse( + { + error: "unauthorized", + message: "Use the private Biologix intelligence bearer token.", + }, + 401, + { "WWW-Authenticate": 'Bearer realm="biologix-intel"' }, + ); + } + + if (request.method === "GET" && route === "/latest") { + return jsonResponse(await store.getLatest()); + } + if (request.method === "GET" && route === "/report") { + return jsonResponse(await store.getReport(clampReportHours(url.searchParams.get("hours")))); + } + if (request.method === "POST" && route === "/snapshot") { + try { + return jsonResponse(await store.runSnapshot("manual"), 201); + } catch (error) { + return jsonResponse( + { + error: "snapshot_failed", + message: error instanceof Error ? error.message : String(error), + }, + 502, + ); + } + } + return jsonResponse({ error: "not_found" }, 404); +} + +export async function runScheduledBiologixSnapshot(env) { + const result = await getStore(env).runSnapshot("cron"); + console.log( + JSON.stringify({ + event: "biologix_public_intel_snapshot", + ...result, + }), + ); + return result; +} diff --git a/tools/biologix-public-intel/cloud/src/worker.js b/tools/biologix-public-intel/cloud/src/worker.js new file mode 100644 index 0000000..da3306c --- /dev/null +++ b/tools/biologix-public-intel/cloud/src/worker.js @@ -0,0 +1,30 @@ +import { + BiologixIntelStore, + handleBiologixIntelRequest, + runScheduledBiologixSnapshot, +} from "./biologix-intel.js"; + +export { BiologixIntelStore }; + +export default { + async fetch(request, env) { + const pathname = new URL(request.url).pathname; + if ( + pathname === "/api/biologix-intel" || + pathname.startsWith("/api/biologix-intel/") + ) { + return handleBiologixIntelRequest(request, env); + } + return new Response("Not found", { + status: 404, + headers: { + "Cache-Control": "no-store", + "Content-Type": "text/plain; charset=utf-8", + "X-Robots-Tag": "noindex, nofollow, noarchive", + }, + }); + }, + async scheduled(controller, env, ctx) { + ctx.waitUntil(runScheduledBiologixSnapshot(env)); + }, +}; diff --git a/tools/biologix-public-intel/cloud/test/biologix-intel.test.mjs b/tools/biologix-public-intel/cloud/test/biologix-intel.test.mjs new file mode 100644 index 0000000..848116e --- /dev/null +++ b/tools/biologix-public-intel/cloud/test/biologix-intel.test.mjs @@ -0,0 +1,168 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + aggregateReport, + buildObservations, + clusterProbableBaskets, + detectPublicTechnology, + detectTrackers, + diffObservations, + inventorySummary, + parseSitemapIndex, + parseStockQuantity, + summarizeUrlset, +} from "../src/biologix-intel-core.js"; + +function product(overrides = {}) { + return { + id: 1, + parent: 0, + name: "Example", + type: "simple", + variation: "", + sku: "SKU-1", + prices: { price: "1000", regular_price: "1000", sale_price: "1000" }, + stock_availability: { text: "10 in stock" }, + is_in_stock: true, + is_on_backorder: false, + is_purchasable: true, + permalink: "https://example.com/product/example", + ...overrides, + }; +} + +test("stock parsing distinguishes exact, unknown, and out of stock", () => { + assert.equal(parseStockQuantity("17 in stock (can be backordered)", true), 17); + assert.equal(parseStockQuantity("In stock", true), null); + assert.equal(parseStockQuantity("Out of stock", false), 0); +}); + +test("variable parent stock is not double-counted when children are exact", () => { + const parent = product({ + id: 10, + type: "variable", + stock_availability: { text: "20 in stock" }, + }); + const variation = product({ + id: 11, + parent: 10, + type: "variation", + variation: "Amount: 10mg", + stock_availability: { text: "7 in stock" }, + }); + const observations = buildObservations([parent], [variation], []); + const summary = inventorySummary(observations); + assert.equal(summary.exact_inventory_units, 7); + assert.equal(summary.displayed_inventory_value_cents, 7000); + assert.equal(observations.find((item) => item.key === "product:10").track_inventory, false); +}); + +test("inventory deltas and synchronized timestamps form an inferred basket", () => { + const previous = buildObservations([product()], [], [ + { id: 1, modified_gmt: "2026-07-25T00:00:00Z" }, + ]); + const current = buildObservations( + [ + product({ stock_availability: { text: "9 in stock" } }), + product({ + id: 2, + sku: "SKU-2", + name: "Second", + prices: { price: "2000" }, + stock_availability: { text: "4 in stock" }, + }), + ], + [], + [ + { id: 1, modified_gmt: "2026-07-25T00:05:01Z" }, + { id: 2, modified_gmt: "2026-07-25T00:05:04Z" }, + ], + ); + const previousMap = Object.fromEntries(previous.map((item) => [item.key, item])); + previousMap["product:2"] = { + ...current.find((item) => item.key === "product:2"), + stock_quantity: 5, + }; + const events = diffObservations( + previousMap, + current, + "2026-07-25T00:05:10Z", + ); + const baskets = clusterProbableBaskets( + events, + "2026-07-25T00:00:00Z", + "2026-07-25T00:05:10Z", + ); + assert.equal(events.filter((event) => event.event_type === "inventory_decrease").length, 2); + assert.equal(baskets.length, 1); + assert.equal(baskets[0].unit_count, 2); + assert.equal(baskets[0].displayed_value_cents, 3000); +}); + +test("tracker and public technology detection is deterministic", () => { + const html = ` + + + + + `; + assert.deepEqual(detectTrackers(html), [ + { provider: "google_analytics", public_id: "G-ABCDEF12" }, + { provider: "meta_pixel", public_id: "123456789" }, + ]); + assert.deepEqual( + detectPublicTechnology(html, { server: "cloudflare", platform: "hostinger" }), + { + technologies: ["cloudflare", "hostinger", "woocommerce", "wordpress"], + plugins: ["woocommerce"], + themes: ["woostify"], + }, + ); +}); + +test("sitemap parsers count public pages without retaining page contents", () => { + const index = ` + + https://example.com/a.xml2026-01-01 + https://example.com/b.xml + + `; + assert.equal(parseSitemapIndex(index).length, 2); + const urlset = ` + + https://example.com/a2026-01-01 + https://example.com/b2026-02-01 + + `; + assert.deepEqual(summarizeUrlset(urlset), { + url_count: 2, + latest_lastmod: "2026-02-01", + }); +}); + +test("reports retain evidence boundaries and never call inventory a paid sale", () => { + const state = { + latest_snapshot: { captured_at: "2026-07-25T01:00:00Z" }, + public_site_signals: { traffic_truth: { visitor_counts_publicly_available: false } }, + }; + const days = [ + { + snapshots: [{ captured_at: "2026-07-25T01:00:00Z" }], + events: [ + { + observed_at: "2026-07-25T01:00:00Z", + event_type: "inventory_decrease", + quantity_delta: -2, + displayed_value_cents: 5000, + }, + ], + baskets: [], + }, + ]; + const report = aggregateReport(days, state, "2026-07-25T00:00:00Z"); + assert.equal(report.movement.observed_units_down, 2); + assert.equal(report.movement.displayed_price_gmv_signal_cents, 5000); + assert.equal(report.traffic.direct_visitor_counts_available, false); + assert.match(report.evidence_boundary.unavailable, /cannot prove payment/); +}); diff --git a/tools/biologix-public-intel/cloud/wrangler.jsonc b/tools/biologix-public-intel/cloud/wrangler.jsonc new file mode 100644 index 0000000..8c039c1 --- /dev/null +++ b/tools/biologix-public-intel/cloud/wrangler.jsonc @@ -0,0 +1,32 @@ +{ + "$schema": "./node_modules/wrangler/config-schema.json", + "name": "biologix-public-intel", + "compatibility_date": "2026-07-22", + "compatibility_flags": [ + "nodejs_compat" + ], + "main": "src/worker.js", + "triggers": { + "crons": [ + "*/15 * * * *" + ] + }, + "observability": { + "enabled": true, + "head_sampling_rate": 1 + }, + "durable_objects": { + "bindings": [ + { + "name": "BIOLOGIX_INTEL", + "class_name": "BiologixIntelStore" + } + ] + }, + "exports": { + "BiologixIntelStore": { + "type": "durable-object", + "storage": "sqlite" + } + } +} From 18f4b579150d4c689c99df578f4d2a54cead3e18 Mon Sep 17 00:00:00 2001 From: Alex Weinstein Date: Fri, 24 Jul 2026 18:20:54 -0700 Subject: [PATCH 3/6] chore: lock Cloudflare worker dependencies --- .../cloud/package-lock.json | 1545 +++++++++++++++++ .../biologix-public-intel/cloud/package.json | 2 +- 2 files changed, 1546 insertions(+), 1 deletion(-) create mode 100644 tools/biologix-public-intel/cloud/package-lock.json diff --git a/tools/biologix-public-intel/cloud/package-lock.json b/tools/biologix-public-intel/cloud/package-lock.json new file mode 100644 index 0000000..7ee83b9 --- /dev/null +++ b/tools/biologix-public-intel/cloud/package-lock.json @@ -0,0 +1,1545 @@ +{ + "name": "biologix-public-intel-cloud", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "biologix-public-intel-cloud", + "version": "1.0.0", + "devDependencies": { + "wrangler": "4.114.0" + }, + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/@cloudflare/kv-asset-handler": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.5.0.tgz", + "integrity": "sha512-jxQYkj8dSIzc0cD6cMMNdOc1UVjqSqu8BZdor5s8cGjW2I8BjODt/kWPVdY+u9zj3ms75Q5qaZgnxUad83+eAg==", + "dev": true, + "license": "MIT OR Apache-2.0", + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@cloudflare/unenv-preset": { + "version": "2.16.1", + "resolved": "https://registry.npmjs.org/@cloudflare/unenv-preset/-/unenv-preset-2.16.1.tgz", + "integrity": "sha512-ECxObrMfyTl5bhQf/lZCXwo5G6xX9IAUo+nDMKK4SZ8m4Jvvxp52vilxyySSWh2YTZz8+HQ07qGH/2rEom1vDw==", + "dev": true, + "license": "MIT OR Apache-2.0", + "peerDependencies": { + "unenv": "2.0.0-rc.24", + "workerd": ">1.20260305.0 <2.0.0-0" + }, + "peerDependenciesMeta": { + "workerd": { + "optional": true + } + } + }, + "node_modules/@cloudflare/workerd-darwin-64": { + "version": "1.20260722.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20260722.1.tgz", + "integrity": "sha512-vZOP8vIS3NwnuaO+gz0FZ7kIGeiO3bZmxV35Ph9zOXKSREhDFlH7wQ7mkCdhW3O4jnXsew+XT7b+DNEI2CcJGQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-darwin-arm64": { + "version": "1.20260722.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20260722.1.tgz", + "integrity": "sha512-EmIQymihDq6WNdER4+LF8Qn80yqayBUpJ+tkOO7wmY8pmgfyXjIUFNXotl21AHovTeu2seR7HdVUgeN/BilCWw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-linux-64": { + "version": "1.20260722.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20260722.1.tgz", + "integrity": "sha512-jvZ3k9fxcnEn04s80CgIYxQfpOyAiz/8qC42DP8EBa9tR27qWyg9wmm31zIobVlrgBZn/+8NfdP73avRGcQOjQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-linux-arm64": { + "version": "1.20260722.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20260722.1.tgz", + "integrity": "sha512-BOSB55SMNdy+DA5uj2WirgiNanpHGis5PVvXH1wSfvjRKr4JGgWK+EZzxz0RFUo6QjjQQC/NimEzNZ7va7jmKg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-windows-64": { + "version": "1.20260722.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20260722.1.tgz", + "integrity": "sha512-sYM8YgUpKnRz2xjvdJLX1Ojzoi4MlA4gk8WTTExhGydjYB2UTs5NIbv0ZmpKgMoK9io3ixgmiW56ZnTbcWOdiA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.11.2", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.2.tgz", + "integrity": "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", + "integrity": "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.1.tgz", + "integrity": "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.1.tgz", + "integrity": "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.1.tgz", + "integrity": "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.1.tgz", + "integrity": "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.1.tgz", + "integrity": "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.1.tgz", + "integrity": "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.1.tgz", + "integrity": "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.1.tgz", + "integrity": "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.1.tgz", + "integrity": "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.1.tgz", + "integrity": "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.1.tgz", + "integrity": "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.1.tgz", + "integrity": "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.1.tgz", + "integrity": "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.1.tgz", + "integrity": "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.1.tgz", + "integrity": "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.1.tgz", + "integrity": "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.1.tgz", + "integrity": "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.1.tgz", + "integrity": "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.1.tgz", + "integrity": "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.1.tgz", + "integrity": "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.1.tgz", + "integrity": "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.1.tgz", + "integrity": "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.1.tgz", + "integrity": "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.1.tgz", + "integrity": "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.1.tgz", + "integrity": "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/colour": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", + "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.35.2.tgz", + "integrity": "sha512-eEieHsMksAW4IiO5NzauESRl2D2qz3J/kwUxUrSfV06A93eEaRfMpHXyUb1mAqrR7i8U9A0GRqE9pjn6u1Jjpg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.3.1" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.35.2.tgz", + "integrity": "sha512-BaktuGPCeHJMARpodR8jK4uKiZrPAy9WrfQW0sdI37clracq8Bp01AYS3SZgi5FS/y5twa9t4+LIuuxQjqRrWw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.3.1" + } + }, + "node_modules/@img/sharp-freebsd-wasm32": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.2.tgz", + "integrity": "sha512-YoAxdnd8hPUkvLHd3bWY+YA8nw3xM/RyRopYucNsWHVSan8NLVM3X2volsfoRDcXdUJPg6tXahSd7HXPK7lRnw==", + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "dependencies": { + "@img/sharp-wasm32": "0.35.2" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.3.1.tgz", + "integrity": "sha512-4V/M3roRMTYjiwZY9IOVQOE8OyeCxFAkYmyZDrZl51uOKjibm3oeEJ4WAmLxutAfzFbC9jqUiPs2gbnGflH+7g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.3.1.tgz", + "integrity": "sha512-c0/DxItpJv2+dGhgycJBBgotdqruGYDvA79drdh0MD1dFpy7JzJ/PlXwi1H4rFf0eTy8tgbI91aHDnZIceY3jQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.3.1.tgz", + "integrity": "sha512-aGGy9aWzXgHBG7HNyQPWorZthlp7+x6fDRoPAQbGO3ThcttuTyKIx3NuSHb6zb4gBNq6/yNn9f1cy9nFKS/Vmg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.3.1.tgz", + "integrity": "sha512-JznefmcK9j1JKPz8AkQDh89kjojubyfOasWBPKfzMIhPwsgDy9evpE/naJTXXXmghS1iFwR8u/kTwh/I2/+GCw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.3.1.tgz", + "integrity": "sha512-1EkwGNCZk6iWNCMWqrvdJ+r1j0PT1zIz60CNPhYnJlK/zyeWqlsPZIe+ocBVqPF8k/Ssee/NCk+tE9Ryrko6ng==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.3.1.tgz", + "integrity": "sha512-Ilays+w2bXdnxzxtQdmXR62u8o8GYa3eL4+Gr+1KiE4xperMZUslRaVPJwwPkzlHEjGfXAfRVAa/7CYCtSqsBw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.3.1.tgz", + "integrity": "sha512-VfBwVHQTbRoj4XlpA/KLZ7ltgMpz+4WSejFzQ+GnoImjo1PtEJ59QB2qR1xQEeRPYIkNrPIm2L4cICMvz4C2ew==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.3.1.tgz", + "integrity": "sha512-+c8ukgwU62DS54nCAjw7keOfHUkmr0B5QHEdcOqRnodF/MNXJbVI8Eopoj4B/0H8Asr65I+A4Amrn7a85/md6A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.3.1.tgz", + "integrity": "sha512-qlKb/pwbkAi1WMsJrYHk7CuDrd12s27U2QnRhFYUoJNrRCmkosMTttuRFat/DDB3IlDm5qE1TJgZ4JDnHX8Ldw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.3.1.tgz", + "integrity": "sha512-yO21HwoUVLN8Qa+/SBjQLMYwBWAVJjeGPNe+hc0OUeMeifEtJqu5a1c4HayE1nNpDih9y3/KkoltfkDodmKAlg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.35.2.tgz", + "integrity": "sha512-SE4kzF2mepn6z+6E7L6lsV8FzuLL6IPQdyX8ZiwROAG/G8td+hP/m7FsFPwidtrF19gvajuC9l6TxAVcsA4S7A==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.3.1" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.35.2.tgz", + "integrity": "sha512-af12Pnd0ZGu2HfP8NayB0kk6eC/lrfbQE6HlR4jD+34wdJ1Vw9TF6TMn6ZvffT+WgqVsl0hRbmNvz2u/23VmwA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.3.1" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.35.2.tgz", + "integrity": "sha512-hYSBm7zcNtDCozCxQHYZJiu63b/bXsgRZuOxCIBZsStMM9Vap47iFHdbX4kCvQsblPB/k+clhELpdQJHQLSHvg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.3.1" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.35.2.tgz", + "integrity": "sha512-qQt0Kc13+Hoan/Awq/qMSQw3L+RI1NCRPgD5cUJ/1WSSmIoysLOc72jlRM3E0OHN9Yr313jgeQ2T+zW+F03QFA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.3.1" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.35.2.tgz", + "integrity": "sha512-E4fLLfRPzDLlEeDaTzI98OFLcv++WL5ChLLMwPoVd0CIoZQqupBSNbOisPL5am9XsbQ9T84+iiMpUvbFtkunbA==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.3.1" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.35.2.tgz", + "integrity": "sha512-gi0zFJJRLswfCZmHtJdikXPOc5u7qamSOS3NHedLqLd4W8Q0NqjdBr6TTRIgsfFjqfTsHFgdfvJ9LwqSgcHiAA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.3.1" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.35.2.tgz", + "integrity": "sha512-siWbOW1u6HFnFLrp0waKyW7VEf7jYvcDWdrXEFa8AkdAQgEvuu5Fz8/Y70w9EeqAdwDtfU012BhEHHaDqvQNzg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.3.1" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.35.2.tgz", + "integrity": "sha512-YBqMMcjDi4QGYiSn4vNOYBhmlC4z5AXqkOUUqI2e0AFA4urNv4ESgOgwNl3K+4etQhha0twXlzeF20bbULm9Yg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.3.1" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.2.tgz", + "integrity": "sha512-Mrv4JQNYVQ94xH+jzZ9r+gowleN8mv2FTgKT+PI6bx5C0G8TdNYndu161pg2i7uoBwxy2ImPMHrJOM2LZef7Bw==", + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.11.1" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-webcontainers-wasm32": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-webcontainers-wasm32/-/sharp-webcontainers-wasm32-0.35.2.tgz", + "integrity": "sha512-QNV27pxs9wpApEiCfvHM1RDoP1w1+2KrUWWDPEhEwg+latvOrfuhWrHWZKwdSFwU6jh3myjw/yOCRsUIuOft3g==", + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@img/sharp-wasm32": "0.35.2" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.35.2.tgz", + "integrity": "sha512-BiVRYc/t6/Vl3e1hBx0hugG4oN9Pydf4fgMSpxTQJmwGUg/YoXTWHiFeRymHfCZzifxu4F4rpk/I67D0LQ20wQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.35.2.tgz", + "integrity": "sha512-YYEhx9PImCC7T0tI8JDMi4DB9LwLCXCU5OWNYEXAxh5Q1ShKkyC6byxzoBJ3gEFDnH2lQckWuDe70G7mB2XJog==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.35.2.tgz", + "integrity": "sha512-imoOyBcoM/iiUr4J6VPpCNjPnjvP/Gks95898yB8YqoGGYmHYbOyCuNv9FMhFgtaiHFGbHW8bxKqRV6VjtXThQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "node_modules/@poppinss/colors": { + "version": "4.1.6", + "resolved": "https://registry.npmjs.org/@poppinss/colors/-/colors-4.1.6.tgz", + "integrity": "sha512-H9xkIdFswbS8n1d6vmRd8+c10t2Qe+rZITbbDHHkQixH5+2x1FDGmi/0K+WgWiqQFKPSlIYB7jlH6Kpfn6Fleg==", + "dev": true, + "license": "MIT", + "dependencies": { + "kleur": "^4.1.5" + } + }, + "node_modules/@poppinss/dumper": { + "version": "0.6.5", + "resolved": "https://registry.npmjs.org/@poppinss/dumper/-/dumper-0.6.5.tgz", + "integrity": "sha512-NBdYIb90J7LfOI32dOewKI1r7wnkiH6m920puQ3qHUeZkxNkQiFnXVWoE6YtFSv6QOiPPf7ys6i+HWWecDz7sw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@poppinss/colors": "^4.1.5", + "@sindresorhus/is": "^7.0.2", + "supports-color": "^10.0.0" + } + }, + "node_modules/@poppinss/exception": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@poppinss/exception/-/exception-1.2.3.tgz", + "integrity": "sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@sindresorhus/is": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-7.2.0.tgz", + "integrity": "sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sindresorhus/is?sponsor=1" + } + }, + "node_modules/@speed-highlight/core": { + "version": "1.2.17", + "resolved": "https://registry.npmjs.org/@speed-highlight/core/-/core-1.2.17.tgz", + "integrity": "sha512-Z92FwKpCtfaW1V0jTU/fh3QzYEZN8wDwrzRIBoADCJfn4mJCNcJN/XegifX7BDrQ8/h9Xh/JnbyMchL0FqXrkg==", + "dev": true, + "license": "CC0-1.0" + }, + "node_modules/blake3-wasm": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/blake3-wasm/-/blake3-wasm-2.1.5.tgz", + "integrity": "sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==", + "dev": true, + "license": "MIT" + }, + "node_modules/cookie": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", + "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/error-stack-parser-es": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/error-stack-parser-es/-/error-stack-parser-es-1.0.5.tgz", + "integrity": "sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/esbuild": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", + "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.1", + "@esbuild/android-arm": "0.28.1", + "@esbuild/android-arm64": "0.28.1", + "@esbuild/android-x64": "0.28.1", + "@esbuild/darwin-arm64": "0.28.1", + "@esbuild/darwin-x64": "0.28.1", + "@esbuild/freebsd-arm64": "0.28.1", + "@esbuild/freebsd-x64": "0.28.1", + "@esbuild/linux-arm": "0.28.1", + "@esbuild/linux-arm64": "0.28.1", + "@esbuild/linux-ia32": "0.28.1", + "@esbuild/linux-loong64": "0.28.1", + "@esbuild/linux-mips64el": "0.28.1", + "@esbuild/linux-ppc64": "0.28.1", + "@esbuild/linux-riscv64": "0.28.1", + "@esbuild/linux-s390x": "0.28.1", + "@esbuild/linux-x64": "0.28.1", + "@esbuild/netbsd-arm64": "0.28.1", + "@esbuild/netbsd-x64": "0.28.1", + "@esbuild/openbsd-arm64": "0.28.1", + "@esbuild/openbsd-x64": "0.28.1", + "@esbuild/openharmony-arm64": "0.28.1", + "@esbuild/sunos-x64": "0.28.1", + "@esbuild/win32-arm64": "0.28.1", + "@esbuild/win32-ia32": "0.28.1", + "@esbuild/win32-x64": "0.28.1" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/kleur": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", + "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/miniflare": { + "version": "4.20260722.0", + "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-4.20260722.0.tgz", + "integrity": "sha512-LW6ABMhCx/yIEFBLC/DO4yAhdm2T/G7jp7pr5T2kj895+CCIaHZqpMXdW9O6YE48LcYcCJChwWc8aEs1vpbTXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cspotcode/source-map-support": "0.8.1", + "sharp": "0.35.2", + "undici": "7.28.0", + "workerd": "1.20260722.1", + "ws": "8.21.0", + "youch": "4.1.0-beta.10" + }, + "bin": { + "miniflare": "bootstrap.js" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/path-to-regexp": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", + "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "dev": true, + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/sharp": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.2.tgz", + "integrity": "sha512-FVtFjtBCMiJS6yb5CX7Sop45WFMpeGw6oRKuJnXYgf/f1ms/D7LE/ZUSNxnW7rZ/dbslQWYkoqFHGPaDBtaK4w==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@img/colour": "^1.1.0", + "detect-libc": "^2.1.2", + "semver": "^7.8.4" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.35.2", + "@img/sharp-darwin-x64": "0.35.2", + "@img/sharp-freebsd-wasm32": "0.35.2", + "@img/sharp-libvips-darwin-arm64": "1.3.1", + "@img/sharp-libvips-darwin-x64": "1.3.1", + "@img/sharp-libvips-linux-arm": "1.3.1", + "@img/sharp-libvips-linux-arm64": "1.3.1", + "@img/sharp-libvips-linux-ppc64": "1.3.1", + "@img/sharp-libvips-linux-riscv64": "1.3.1", + "@img/sharp-libvips-linux-s390x": "1.3.1", + "@img/sharp-libvips-linux-x64": "1.3.1", + "@img/sharp-libvips-linuxmusl-arm64": "1.3.1", + "@img/sharp-libvips-linuxmusl-x64": "1.3.1", + "@img/sharp-linux-arm": "0.35.2", + "@img/sharp-linux-arm64": "0.35.2", + "@img/sharp-linux-ppc64": "0.35.2", + "@img/sharp-linux-riscv64": "0.35.2", + "@img/sharp-linux-s390x": "0.35.2", + "@img/sharp-linux-x64": "0.35.2", + "@img/sharp-linuxmusl-arm64": "0.35.2", + "@img/sharp-linuxmusl-x64": "0.35.2", + "@img/sharp-webcontainers-wasm32": "0.35.2", + "@img/sharp-win32-arm64": "0.35.2", + "@img/sharp-win32-ia32": "0.35.2", + "@img/sharp-win32-x64": "0.35.2" + } + }, + "node_modules/supports-color": { + "version": "10.2.2", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-10.2.2.tgz", + "integrity": "sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD", + "optional": true + }, + "node_modules/undici": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz", + "integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, + "node_modules/unenv": { + "version": "2.0.0-rc.24", + "resolved": "https://registry.npmjs.org/unenv/-/unenv-2.0.0-rc.24.tgz", + "integrity": "sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "pathe": "^2.0.3" + } + }, + "node_modules/workerd": { + "version": "1.20260722.1", + "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260722.1.tgz", + "integrity": "sha512-NycKuc1x2onvsRfGGpM093vRlLFU2zHDAM0+APpccfg4+gZxDGCH27RmdDvkeBuoZyYqgLo3oAfF6re4mvC3vQ==", + "dev": true, + "hasInstallScript": true, + "license": "Apache-2.0", + "bin": { + "workerd": "bin/workerd" + }, + "engines": { + "node": ">=16" + }, + "optionalDependencies": { + "@cloudflare/workerd-darwin-64": "1.20260722.1", + "@cloudflare/workerd-darwin-arm64": "1.20260722.1", + "@cloudflare/workerd-linux-64": "1.20260722.1", + "@cloudflare/workerd-linux-arm64": "1.20260722.1", + "@cloudflare/workerd-windows-64": "1.20260722.1" + } + }, + "node_modules/wrangler": { + "version": "4.114.0", + "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-4.114.0.tgz", + "integrity": "sha512-M65P25t5UHA1TIJfgZXDcj+YzVobgKdRguM2QPz0xnxLFuOcuE3ErgllDht0iaho7MS4o0g/Bb4YK2+GT+bibg==", + "dev": true, + "license": "MIT OR Apache-2.0", + "dependencies": { + "@cloudflare/kv-asset-handler": "0.5.0", + "@cloudflare/unenv-preset": "2.16.1", + "blake3-wasm": "2.1.5", + "esbuild": "0.28.1", + "miniflare": "4.20260722.0", + "path-to-regexp": "6.3.0", + "unenv": "2.0.0-rc.24", + "workerd": "1.20260722.1" + }, + "bin": { + "cf-wrangler": "bin/cf-wrangler.js", + "wrangler": "bin/wrangler.js", + "wrangler2": "bin/wrangler.js" + }, + "engines": { + "node": ">=22.0.0" + }, + "optionalDependencies": { + "fsevents": "2.3.3" + }, + "peerDependencies": { + "@cloudflare/workers-types": "^5.20260722.1" + }, + "peerDependenciesMeta": { + "@cloudflare/workers-types": { + "optional": true + } + } + }, + "node_modules/ws": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/youch": { + "version": "4.1.0-beta.10", + "resolved": "https://registry.npmjs.org/youch/-/youch-4.1.0-beta.10.tgz", + "integrity": "sha512-rLfVLB4FgQneDr0dv1oddCVZmKjcJ6yX6mS4pU82Mq/Dt9a3cLZQ62pDBL4AUO+uVrCvtWz3ZFUL2HFAFJ/BXQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@poppinss/colors": "^4.1.5", + "@poppinss/dumper": "^0.6.4", + "@speed-highlight/core": "^1.2.7", + "cookie": "^1.0.2", + "youch-core": "^0.3.3" + } + }, + "node_modules/youch-core": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/youch-core/-/youch-core-0.3.3.tgz", + "integrity": "sha512-ho7XuGjLaJ2hWHoK8yFnsUGy2Y5uDpqSTq1FkHLK4/oqKtyUU1AFbOOxY4IpC9f0fTLjwYbslUz0Po5BpD1wrA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@poppinss/exception": "^1.2.2", + "error-stack-parser-es": "^1.0.5" + } + } + } +} diff --git a/tools/biologix-public-intel/cloud/package.json b/tools/biologix-public-intel/cloud/package.json index 6a9def8..cbb68f9 100644 --- a/tools/biologix-public-intel/cloud/package.json +++ b/tools/biologix-public-intel/cloud/package.json @@ -9,7 +9,7 @@ "validate": "npm test && wrangler deploy --dry-run --outdir .dry-run" }, "devDependencies": { - "wrangler": "4.113.0" + "wrangler": "4.114.0" }, "engines": { "node": ">=20.19.0" From 4987c26c872a91e2fd2d3b997af149b70e2ef648 Mon Sep 17 00:00:00 2001 From: Alex Weinstein Date: Fri, 24 Jul 2026 18:53:17 -0700 Subject: [PATCH 4/6] feat: deploy poller on Vercel Cron --- tools/biologix-public-intel/README.md | 11 +- tools/biologix-public-intel/vercel/.gitignore | 2 + tools/biologix-public-intel/vercel/README.md | 35 + .../vercel/api/biologix-intel/[route].js | 51 ++ .../biologix-public-intel/vercel/api/cron.js | 28 + .../vercel/lib/biologix-intel-core.js | 561 ++++++++++++++++ .../vercel/lib/blob-store.js | 145 ++++ .../vercel/lib/collector.js | 620 ++++++++++++++++++ .../biologix-public-intel/vercel/lib/http.js | 37 ++ .../biologix-public-intel/vercel/lib/state.js | 128 ++++ .../vercel/package-lock.json | 369 +++++++++++ .../biologix-public-intel/vercel/package.json | 16 + .../vercel/test/biologix-intel.test.mjs | 168 +++++ .../vercel/test/state.test.mjs | 66 ++ .../biologix-public-intel/vercel/vercel.json | 17 + 15 files changed, 2250 insertions(+), 4 deletions(-) create mode 100644 tools/biologix-public-intel/vercel/.gitignore create mode 100644 tools/biologix-public-intel/vercel/README.md create mode 100644 tools/biologix-public-intel/vercel/api/biologix-intel/[route].js create mode 100644 tools/biologix-public-intel/vercel/api/cron.js create mode 100644 tools/biologix-public-intel/vercel/lib/biologix-intel-core.js create mode 100644 tools/biologix-public-intel/vercel/lib/blob-store.js create mode 100644 tools/biologix-public-intel/vercel/lib/collector.js create mode 100644 tools/biologix-public-intel/vercel/lib/http.js create mode 100644 tools/biologix-public-intel/vercel/lib/state.js create mode 100644 tools/biologix-public-intel/vercel/package-lock.json create mode 100644 tools/biologix-public-intel/vercel/package.json create mode 100644 tools/biologix-public-intel/vercel/test/biologix-intel.test.mjs create mode 100644 tools/biologix-public-intel/vercel/test/state.test.mjs create mode 100644 tools/biologix-public-intel/vercel/vercel.json diff --git a/tools/biologix-public-intel/README.md b/tools/biologix-public-intel/README.md index 7b721f8..0b80c98 100644 --- a/tools/biologix-public-intel/README.md +++ b/tools/biologix-public-intel/README.md @@ -145,9 +145,10 @@ and not an accounting system. ## Cloud deployment -The connected Sites Worker runs the cloud collector every 15 minutes. Its -SQLite-backed Durable Object retains 120 days of snapshot summaries, event -deltas, probable-basket inferences, and public site signals. +The production service at +`https://biologix-public-intel.vercel.app` runs every 15 minutes on Vercel +Cron. A private Vercel Blob store retains 120 days of snapshot summaries, +event deltas, probable-basket inferences, and public site signals. The cloud runtime additionally records: @@ -173,4 +174,6 @@ POST /api/biologix-intel/snapshot ``` Only the health endpoint is public. The remaining endpoints require the private -`BIOLOGIX_INTEL_TOKEN` bearer token stored in the Sites production environment. +`INTEL_API_TOKEN` bearer token stored as a sensitive Vercel production variable. +The cron route has a separate `CRON_SECRET`. See `vercel/` for the deployable +service and its verification commands. diff --git a/tools/biologix-public-intel/vercel/.gitignore b/tools/biologix-public-intel/vercel/.gitignore new file mode 100644 index 0000000..6096ed2 --- /dev/null +++ b/tools/biologix-public-intel/vercel/.gitignore @@ -0,0 +1,2 @@ +.vercel +node_modules diff --git a/tools/biologix-public-intel/vercel/README.md b/tools/biologix-public-intel/vercel/README.md new file mode 100644 index 0000000..de017f8 --- /dev/null +++ b/tools/biologix-public-intel/vercel/README.md @@ -0,0 +1,35 @@ +# Biologix public-intelligence service on Vercel + +This internal service runs every 15 minutes on Vercel Cron and stores its +rolling 120-day history in a private Vercel Blob store. + +It performs public, unauthenticated GET requests only. It records inventory, +price, availability, popularity order, product modification timestamps, +sitemap counts, WordPress/WooCommerce surface metadata, tracker/technology +presence, DNS records, cache headers, and response latency. Correlated +inventory decreases are labeled as probable baskets, never confirmed sales. + +It does not collect customers, orders, carts, reviews, cookies, PII, raw HTML, +or authenticated WordPress/WooCommerce data. Visitors, sessions, pageviews, +traffic sources, conversion rate, and paid revenue are not publicly exposed by +the target and therefore remain explicitly unavailable. + +## API + +- `GET /api/biologix-intel/health` is public and contains no sensitive data. +- `GET /api/biologix-intel/latest` requires `Authorization: Bearer ...`. +- `GET /api/biologix-intel/report?hours=24` requires the same bearer token. +- `POST /api/biologix-intel/snapshot` requires the same bearer token. +- `GET /api/cron` is reserved for Vercel Cron and protected by `CRON_SECRET`. + +Production requires `INTEL_API_TOKEN`, `CRON_SECRET`, and the private Blob +store's automatically connected `BLOB_READ_WRITE_TOKEN`. + +## Verification + +```bash +npm ci +npm test +npm run check +vercel build +``` diff --git a/tools/biologix-public-intel/vercel/api/biologix-intel/[route].js b/tools/biologix-public-intel/vercel/api/biologix-intel/[route].js new file mode 100644 index 0000000..67c0e59 --- /dev/null +++ b/tools/biologix-public-intel/vercel/api/biologix-intel/[route].js @@ -0,0 +1,51 @@ +import { + getHealth, + getLatest, + getReport, + runSnapshot, +} from "../../lib/blob-store.js"; +import { hasBearerToken, json, unauthorized } from "../../lib/http.js"; + +export const maxDuration = 300; + +async function handle(request) { + const url = new URL(request.url); + const route = url.pathname.split("/").at(-1); + + try { + if (request.method === "GET" && route === "health") { + return json(await getHealth()); + } + + if (!hasBearerToken(request, process.env.INTEL_API_TOKEN)) { + return unauthorized(); + } + + if (request.method === "GET" && route === "latest") { + return json(await getLatest()); + } + if (request.method === "GET" && route === "report") { + return json(await getReport(url.searchParams.get("hours"))); + } + if (request.method === "POST" && route === "snapshot") { + return json(await runSnapshot("manual"), 201); + } + return json({ error: "not_found" }, 404); + } catch (error) { + return json( + { + error: "intel_request_failed", + message: error instanceof Error ? error.message : String(error), + }, + 502, + ); + } +} + +export function GET(request) { + return handle(request); +} + +export function POST(request) { + return handle(request); +} diff --git a/tools/biologix-public-intel/vercel/api/cron.js b/tools/biologix-public-intel/vercel/api/cron.js new file mode 100644 index 0000000..3898e62 --- /dev/null +++ b/tools/biologix-public-intel/vercel/api/cron.js @@ -0,0 +1,28 @@ +import { runSnapshot } from "../lib/blob-store.js"; +import { hasBearerToken, json, unauthorized } from "../lib/http.js"; + +export const maxDuration = 300; + +export async function GET(request) { + if ( + request.method !== "GET" || + !hasBearerToken(request, process.env.CRON_SECRET) + ) { + return unauthorized(); + } + + try { + const result = await runSnapshot("cron"); + console.log(JSON.stringify({ event: "biologix_public_intel_snapshot", ...result })); + return json(result); + } catch (error) { + console.error(error); + return json( + { + error: "snapshot_failed", + message: error instanceof Error ? error.message : String(error), + }, + 502, + ); + } +} diff --git a/tools/biologix-public-intel/vercel/lib/biologix-intel-core.js b/tools/biologix-public-intel/vercel/lib/biologix-intel-core.js new file mode 100644 index 0000000..5535b2a --- /dev/null +++ b/tools/biologix-public-intel/vercel/lib/biologix-intel-core.js @@ -0,0 +1,561 @@ +export const BIOLOGIX_BASE_URL = "https://biologixlabsresearch.com"; +export const POLL_INTERVAL_MINUTES = 15; +export const DEEP_SCAN_INTERVAL_MS = 6 * 60 * 60 * 1000; +export const RETENTION_DAYS = 120; + +const TRACKER_PATTERNS = [ + ["google_tag_manager", /\bGTM-[A-Z0-9]{5,}\b/gi], + ["google_analytics", /\bG-[A-Z0-9]{6,}\b/gi], + ["universal_analytics", /\bUA-\d{4,}-\d+\b/gi], + ["google_ads", /\bAW-\d{5,}\b/gi], + ["meta_pixel", /fbq\(\s*['"]init['"]\s*,\s*['"](\d{5,})['"]/gi], + ["tiktok_pixel", /ttq\.load\(\s*['"]([A-Z0-9]{8,})['"]/gi], + ["microsoft_clarity", /clarity\.ms\/tag\/([a-z0-9]+)/gi], + ["hotjar", /hjid\s*[:=]\s*(\d+)/gi], +]; + +const GENERIC_TRACKER_MARKERS = [ + ["brevo", ["sibautomation.com", "sendinblue"]], + ["meta_pixel_present", ["connect.facebook.net/en_us/fbevents.js"]], + ["tiktok_pixel_present", ["analytics.tiktok.com"]], + ["optinmonster", ["optinmonster.com", "omappapi.com"]], +]; + +const RELEVANT_NAMESPACE_PATTERNS = [ + /^bankful\//, + /^jetpack\//, + /^linkmoney\//, + /^omapp\//, + /^rankmath\//, + /^sendinblue-woo\//, + /^wc-admin$/, + /^wc-analytics$/, + /^wc-push-notifications$/, + /^wc-telemetry$/, + /^wc\/store/, + /^woocommerce/, + /^wpforms\//, +]; + +const PUBLIC_AGGREGATE_PROBES = [ + ["rankmath_analytics", "/wp-json/rankmath/v1/an/analyticsSummary"], + ["rankmath_dashboard", "/wp-json/rankmath/v1/an/dashboard"], + ["rankmath_keywords", "/wp-json/rankmath/v1/an/keywordsSummary"], + ["rankmath_link_stats", "/wp-json/rankmath/v1/links/links-stats"], + ["rankmath_ai_visibility", "/wp-json/rankmath/v1/ai-visibility/overview"], + ["woocommerce_sales", "/wp-json/wc/v1/reports/sales"], + ["woocommerce_top_sellers", "/wp-json/wc/v1/reports/top_sellers"], + ["woocommerce_revenue", "/wp-json/wc-analytics/reports/revenue/stats"], + ["woocommerce_orders", "/wp-json/wc-analytics/reports/orders/stats"], +]; + +export function parseStockQuantity(stockText, inStock) { + const text = String(stockText ?? "").trim(); + const match = text.match(/^(-?\d+)\s+in stock\b/i); + if (match) return Number.parseInt(match[1], 10); + if (!inStock && text.toLowerCase().startsWith("out of stock")) return 0; + return null; +} + +export function cents(value) { + if (value === null || value === undefined || value === "") return null; + const parsed = Number.parseInt(value, 10); + return Number.isFinite(parsed) ? parsed : null; +} + +export function detectTrackers(html) { + const found = new Map(); + for (const [provider, pattern] of TRACKER_PATTERNS) { + pattern.lastIndex = 0; + for (const match of html.matchAll(pattern)) { + const publicId = String(match[1] ?? match[0]).toUpperCase(); + found.set(`${provider}:${publicId}`, { provider, public_id: publicId }); + } + } + + const lowered = html.toLowerCase(); + for (const [provider, markers] of GENERIC_TRACKER_MARKERS) { + if (markers.some((marker) => lowered.includes(marker))) { + found.set(`${provider}:present`, { provider, public_id: "present" }); + } + } + + return [...found.values()].sort((a, b) => + `${a.provider}:${a.public_id}`.localeCompare(`${b.provider}:${b.public_id}`), + ); +} + +export function detectPublicTechnology(html, headers = {}) { + const lowered = html.toLowerCase(); + const plugins = new Set(); + for (const match of html.matchAll(/\/wp-content\/plugins\/([^/'"?]+)/gi)) { + const slug = match[1].toLowerCase(); + if (/^[a-z0-9][a-z0-9._-]*$/.test(slug)) plugins.add(slug); + } + + const themes = new Set(); + for (const match of html.matchAll(/\/wp-content\/themes\/([^/'"?]+)/gi)) { + themes.add(match[1].toLowerCase()); + } + + const technologies = new Set(["wordpress"]); + if (lowered.includes("woocommerce")) technologies.add("woocommerce"); + if (lowered.includes("elementor")) technologies.add("elementor"); + if (lowered.includes("rank-math")) technologies.add("rank-math"); + if (lowered.includes("litespeed")) technologies.add("litespeed"); + if (String(headers.server ?? "").toLowerCase().includes("cloudflare")) { + technologies.add("cloudflare"); + } + if (String(headers.platform ?? "").toLowerCase().includes("hostinger")) { + technologies.add("hostinger"); + } + + return { + technologies: [...technologies].sort(), + plugins: [...plugins].sort(), + themes: [...themes].sort(), + }; +} + +function normalizeObservation(item, options) { + const prices = item.prices ?? {}; + const stockText = String(item.stock_availability?.text ?? ""); + const productId = Number(item.id); + const parentId = Number(item.parent || productId); + return { + key: `${options.recordType}:${productId}`, + record_type: options.recordType, + product_type: String(item.type ?? ""), + product_id: productId, + parent_id: parentId, + name: String(item.name ?? ""), + variation: String(item.variation ?? ""), + sku: String(item.sku ?? ""), + price_cents: cents(prices.price), + regular_price_cents: cents(prices.regular_price), + sale_price_cents: cents(prices.sale_price), + stock_quantity: options.quantity, + stock_text: stockText, + in_stock: Boolean(item.is_in_stock), + on_backorder: Boolean(item.is_on_backorder), + purchasable: Boolean(item.is_purchasable), + track_inventory: Boolean(options.trackInventory), + popularity_rank: options.rank ?? null, + modified_gmt: options.modifiedById.get(parentId) ?? null, + permalink: String(item.permalink ?? ""), + }; +} + +export function buildObservations(parents, variations, wpProducts = []) { + const modifiedById = new Map( + wpProducts + .filter((product) => product?.id !== undefined) + .map((product) => [Number(product.id), product.modified_gmt ?? null]), + ); + const variationQuantities = new Map(); + const exactChildParents = new Set(); + + for (const variation of variations) { + const quantity = parseStockQuantity( + variation.stock_availability?.text, + Boolean(variation.is_in_stock), + ); + variationQuantities.set(Number(variation.id), quantity); + if (quantity !== null) exactChildParents.add(Number(variation.parent || 0)); + } + + const observations = []; + parents.forEach((parent, index) => { + const quantity = parseStockQuantity( + parent.stock_availability?.text, + Boolean(parent.is_in_stock), + ); + const productId = Number(parent.id); + const productType = String(parent.type ?? ""); + const trackInventory = + quantity !== null && + Boolean(parent.is_purchasable) && + (productType === "simple" || + (productType === "variable" && !exactChildParents.has(productId))); + observations.push( + normalizeObservation(parent, { + recordType: "product", + rank: index + 1, + quantity, + trackInventory, + modifiedById, + }), + ); + }); + + for (const variation of variations) { + const quantity = variationQuantities.get(Number(variation.id)); + observations.push( + normalizeObservation(variation, { + recordType: "variation", + rank: null, + quantity, + trackInventory: quantity !== null && Boolean(variation.is_purchasable), + modifiedById, + }), + ); + } + return observations; +} + +export function inventorySummary(observations) { + const tracked = observations.filter( + (item) => item.track_inventory && item.stock_quantity !== null, + ); + return { + exact_inventory_units: tracked.reduce( + (total, item) => total + (item.stock_quantity ?? 0), + 0, + ), + displayed_inventory_value_cents: tracked.reduce( + (total, item) => + total + (item.stock_quantity ?? 0) * (item.price_cents ?? 0), + 0, + ), + exact_quantity_records: tracked.length, + positive_stock_records: tracked.filter((item) => item.stock_quantity > 0).length, + zero_stock_records: tracked.filter((item) => item.stock_quantity === 0).length, + hidden_purchasable_quantity_records: observations.filter( + (item) => + item.record_type === "variation" && + item.purchasable && + item.stock_quantity === null, + ).length, + backorder_capable_records: observations.filter((item) => + item.stock_text.toLowerCase().includes("can be backordered"), + ).length, + max_exact_stock_quantity: tracked.reduce( + (maximum, item) => Math.max(maximum, item.stock_quantity ?? 0), + 0, + ), + }; +} + +function eventId(capturedAt, eventType, key) { + return `${capturedAt}:${eventType}:${key}`; +} + +export function diffObservations(previousByKey, current, capturedAt) { + if (!previousByKey || Object.keys(previousByKey).length === 0) return []; + const events = []; + + for (const item of current) { + const previous = previousByKey[item.key]; + if (!previous) { + events.push({ + id: eventId(capturedAt, "catalog_added", item.key), + observed_at: capturedAt, + event_type: "catalog_added", + item_key: item.key, + parent_id: item.parent_id, + name: item.name, + variation: item.variation, + sku: item.sku, + evidence_level: "observed", + }); + continue; + } + + if ( + previous.track_inventory && + item.track_inventory && + previous.stock_quantity !== null && + item.stock_quantity !== null && + previous.stock_quantity !== item.stock_quantity + ) { + const delta = item.stock_quantity - previous.stock_quantity; + events.push({ + id: eventId(capturedAt, "inventory", item.key), + observed_at: capturedAt, + event_type: delta < 0 ? "inventory_decrease" : "inventory_increase", + item_key: item.key, + parent_id: item.parent_id, + name: item.name, + variation: item.variation, + sku: item.sku, + old_value: previous.stock_quantity, + new_value: item.stock_quantity, + quantity_delta: delta, + price_cents: item.price_cents, + displayed_value_cents: + delta < 0 ? Math.abs(delta) * (item.price_cents ?? 0) : 0, + modified_gmt: item.modified_gmt, + evidence_level: "observed", + }); + } + + if (previous.price_cents !== item.price_cents) { + events.push({ + id: eventId(capturedAt, "price", item.key), + observed_at: capturedAt, + event_type: "price_change", + item_key: item.key, + parent_id: item.parent_id, + name: item.name, + variation: item.variation, + sku: item.sku, + old_value: previous.price_cents, + new_value: item.price_cents, + evidence_level: "observed", + }); + } + + if ( + item.record_type === "product" && + previous.popularity_rank !== item.popularity_rank + ) { + events.push({ + id: eventId(capturedAt, "rank", item.key), + observed_at: capturedAt, + event_type: "popularity_rank_change", + item_key: item.key, + parent_id: item.parent_id, + name: item.name, + variation: "", + old_value: previous.popularity_rank, + new_value: item.popularity_rank, + evidence_level: "observed", + }); + } + + if (previous.in_stock !== item.in_stock) { + events.push({ + id: eventId(capturedAt, "availability", item.key), + observed_at: capturedAt, + event_type: "availability_change", + item_key: item.key, + parent_id: item.parent_id, + name: item.name, + variation: item.variation, + old_value: previous.in_stock, + new_value: item.in_stock, + evidence_level: "observed", + }); + } + } + + const currentKeys = new Set(current.map((item) => item.key)); + for (const previous of Object.values(previousByKey)) { + if (!currentKeys.has(previous.key)) { + events.push({ + id: eventId(capturedAt, "catalog_removed", previous.key), + observed_at: capturedAt, + event_type: "catalog_removed", + item_key: previous.key, + parent_id: previous.parent_id, + name: previous.name, + variation: previous.variation, + sku: previous.sku, + evidence_level: "observed", + }); + } + } + return events; +} + +export function clusterProbableBaskets(events, previousCapturedAt, capturedAt) { + const windowStart = Date.parse(previousCapturedAt ?? "") || 0; + const windowEnd = Date.parse(capturedAt) + 30_000; + const candidates = events + .filter((event) => { + if (event.event_type !== "inventory_decrease" || !event.modified_gmt) { + return false; + } + const modified = Date.parse(event.modified_gmt); + return modified >= windowStart && modified <= windowEnd; + }) + .sort((a, b) => Date.parse(a.modified_gmt) - Date.parse(b.modified_gmt)); + + const rawGroups = []; + let current = []; + for (const event of candidates) { + if ( + current.length === 0 || + Date.parse(event.modified_gmt) - + Date.parse(current[current.length - 1].modified_gmt) <= + 5_000 + ) { + current.push(event); + } else { + rawGroups.push(current); + current = [event]; + } + } + if (current.length) rawGroups.push(current); + + return rawGroups + .filter((group) => group.length >= 2) + .map((group) => { + const groupId = `basket:${capturedAt}:${group + .map((event) => event.item_key) + .sort() + .join("|")}`; + for (const event of group) event.group_id = groupId; + return { + group_id: groupId, + observed_at: capturedAt, + occurred_at: group[0].modified_gmt, + item_count: group.length, + unit_count: group.reduce( + (total, event) => total + Math.abs(event.quantity_delta ?? 0), + 0, + ), + displayed_value_cents: group.reduce( + (total, event) => total + (event.displayed_value_cents ?? 0), + 0, + ), + confidence: 0.7, + classification: "probable_basket_not_confirmed_sale", + event_ids: group.map((event) => event.id), + }; + }); +} + +export function parseSitemapIndex(xml) { + const entries = []; + const sitemapBlocks = xml.match(//gi) ?? []; + for (const block of sitemapBlocks) { + const location = block.match(/([\s\S]*?)<\/loc>/i)?.[1]?.trim(); + if (!location) continue; + const lastmod = block.match(/([\s\S]*?)<\/lastmod>/i)?.[1]?.trim(); + entries.push({ location, lastmod: lastmod ?? null }); + } + return entries; +} + +export function summarizeUrlset(xml) { + const urlBlocks = xml.match(//gi) ?? []; + const lastmods = urlBlocks + .map((block) => block.match(/([\s\S]*?)<\/lastmod>/i)?.[1]?.trim()) + .filter(Boolean) + .sort(); + return { + url_count: urlBlocks.length, + latest_lastmod: lastmods.at(-1) ?? null, + }; +} + +export function relevantNamespaces(namespaces) { + return [...new Set(namespaces)] + .filter((namespace) => + RELEVANT_NAMESPACE_PATTERNS.some((pattern) => pattern.test(namespace)), + ) + .sort(); +} + +export function publicAggregateProbes() { + return [...PUBLIC_AGGREGATE_PROBES]; +} + +export function aggregateReport(dayRecords, latestState, sinceIso) { + const cutoff = Date.parse(sinceIso); + const snapshots = dayRecords + .flatMap((day) => day?.snapshots ?? []) + .filter((snapshot) => Date.parse(snapshot.captured_at) >= cutoff) + .sort((a, b) => Date.parse(a.captured_at) - Date.parse(b.captured_at)); + const events = dayRecords + .flatMap((day) => day?.events ?? []) + .filter((event) => Date.parse(event.observed_at) >= cutoff); + const baskets = dayRecords + .flatMap((day) => day?.baskets ?? []) + .filter((basket) => Date.parse(basket.observed_at) >= cutoff); + + const decreases = events.filter( + (event) => event.event_type === "inventory_decrease", + ); + const increases = events.filter( + (event) => event.event_type === "inventory_increase", + ); + const latest = latestState?.latest_snapshot ?? null; + return { + generated_at: new Date().toISOString(), + window: { + since: sinceIso, + first_snapshot_at: snapshots[0]?.captured_at ?? null, + last_snapshot_at: snapshots.at(-1)?.captured_at ?? null, + snapshot_count: snapshots.length, + }, + current: latest, + movement: { + observed_units_down: decreases.reduce( + (total, event) => total + Math.abs(event.quantity_delta ?? 0), + 0, + ), + observed_units_up: increases.reduce( + (total, event) => total + Math.abs(event.quantity_delta ?? 0), + 0, + ), + displayed_price_gmv_signal_cents: decreases.reduce( + (total, event) => total + (event.displayed_value_cents ?? 0), + 0, + ), + inventory_decrease_records: decreases.length, + inventory_increase_records: increases.length, + price_changes: events.filter((event) => event.event_type === "price_change") + .length, + popularity_rank_changes: events.filter( + (event) => event.event_type === "popularity_rank_change", + ).length, + availability_changes: events.filter( + (event) => event.event_type === "availability_change", + ).length, + }, + probable_baskets: { + count: baskets.length, + units: baskets.reduce((total, basket) => total + basket.unit_count, 0), + displayed_value_cents: baskets.reduce( + (total, basket) => total + basket.displayed_value_cents, + 0, + ), + classification: "inference_not_confirmed_sale", + confidence: 0.7, + recent: baskets.slice(-50).reverse(), + }, + recent_inventory_events: [...decreases, ...increases] + .sort((a, b) => Date.parse(b.observed_at) - Date.parse(a.observed_at)) + .slice(0, 100), + public_site_signals: latestState?.public_site_signals ?? null, + traffic: { + direct_visitor_counts_available: false, + observed_public_signals: [ + "storefront inventory movement", + "public popularity-order movement", + "public product modification timestamps", + "sitemap growth and modification times", + "public analytics-tag and technology presence", + "origin response timing and cache headers", + ], + unavailable_without_authorized_or_paid_data: [ + "visitors", + "sessions", + "pageviews", + "traffic sources", + "conversion rate", + "paid and settled sales", + "refunds and chargebacks", + ], + }, + evidence_boundary: { + observed: + "Public GET responses and changes between scheduled snapshots.", + inferred: + "Probable baskets use correlated inventory decreases and public modification timestamps.", + unavailable: + "Inventory movement cannot prove payment, settlement, fulfillment, customer identity, or traffic.", + }, + }; +} + +export function compactObservationMap(observations) { + return Object.fromEntries(observations.map((item) => [item.key, item])); +} + +export function safeDurationSince(value, now = Date.now()) { + const timestamp = Date.parse(value ?? ""); + return Number.isFinite(timestamp) ? Math.max(0, now - timestamp) : null; +} diff --git a/tools/biologix-public-intel/vercel/lib/blob-store.js b/tools/biologix-public-intel/vercel/lib/blob-store.js new file mode 100644 index 0000000..6b636cd --- /dev/null +++ b/tools/biologix-public-intel/vercel/lib/blob-store.js @@ -0,0 +1,145 @@ +import { + BlobPreconditionFailedError, + del, + get, + put, +} from "@vercel/blob"; + +import { collectPublicSnapshot } from "./collector.js"; +import { + applySnapshot, + createDay, + createState, + healthForState, + markAttempt, + markFailure, + reportFromDays, + shouldSkipCron, +} from "./state.js"; + +const STATE_PATH = "biologix-intel/state.json"; + +export function normalizeBlobEtag(etag) { + return etag?.replace(/^W\//, "") ?? null; +} + +function dayPath(dateKey) { + return `biologix-intel/days/${dateKey}.json`; +} + +async function readJson(pathname) { + const result = await get(pathname, { access: "private" }); + if (!result) return { value: null, etag: null }; + if (result.statusCode !== 200 || !result.stream) { + throw new Error(`Blob read failed for ${pathname}: ${result.statusCode}`); + } + return { + value: JSON.parse(await new Response(result.stream).text()), + etag: normalizeBlobEtag(result.blob.etag), + }; +} + +async function writeJson(pathname, value, etag) { + return put(pathname, JSON.stringify(value), { + access: "private", + contentType: "application/json", + cacheControlMaxAge: 60, + allowOverwrite: Boolean(etag), + ...(etag ? { ifMatch: etag } : {}), + }); +} + +async function loadState() { + const record = await readJson(STATE_PATH); + return { + state: record.value ?? createState(), + etag: record.etag, + }; +} + +export async function getHealth() { + const { state } = await loadState(); + return healthForState(state); +} + +export async function getLatest() { + const { state } = await loadState(); + return { + health: healthForState(state), + snapshot: state.latest_snapshot, + public_site_signals: state.public_site_signals, + }; +} + +export async function getReport(hours = 24) { + const { state } = await loadState(); + const parsed = Number.parseInt(hours ?? "24", 10); + const clampedHours = Number.isFinite(parsed) + ? Math.min(120 * 24, Math.max(1, parsed)) + : 24; + const cutoff = Date.now() - clampedHours * 60 * 60 * 1000; + const dayKeys = state.day_keys.filter( + (key) => Date.parse(`${key}T23:59:59.999Z`) >= cutoff, + ); + const days = ( + await Promise.all(dayKeys.map(async (key) => (await readJson(dayPath(key))).value)) + ).filter(Boolean); + return reportFromDays(days, state, clampedHours); +} + +export async function runSnapshot(trigger = "cron") { + const loaded = await loadState(); + let state = loaded.state; + let stateEtag = loaded.etag; + if (trigger === "cron" && shouldSkipCron(state)) { + return { + skipped: true, + reason: "duplicate_trigger_guard", + ...healthForState(state), + }; + } + + const attemptedAt = new Date().toISOString(); + state = markAttempt(state, attemptedAt); + try { + stateEtag = (await writeJson(STATE_PATH, state, stateEtag)).etag; + } catch (error) { + if (error instanceof BlobPreconditionFailedError) { + return { + skipped: true, + reason: "concurrent_run_guard", + ...(await getHealth()), + }; + } + throw error; + } + + try { + const result = await collectPublicSnapshot(state, trigger); + const dateKey = result.summary.captured_at.slice(0, 10); + const dayRecord = await readJson(dayPath(dateKey)); + const applied = applySnapshot( + state, + dayRecord.value ?? createDay(dateKey), + result, + ); + await writeJson(dayPath(dateKey), applied.day, dayRecord.etag); + await writeJson(STATE_PATH, applied.state, stateEtag); + if (applied.expiredDayKeys.length) { + await del(applied.expiredDayKeys.map(dayPath)); + } + return { + skipped: false, + snapshot: result.summary, + probable_baskets: result.baskets, + }; + } catch (error) { + const failedState = markFailure(state, error, new Date().toISOString()); + try { + await writeJson(STATE_PATH, failedState, stateEtag); + } catch { + // Preserve the original collection error if another invocation won the race. + } + throw error; + } +} diff --git a/tools/biologix-public-intel/vercel/lib/collector.js b/tools/biologix-public-intel/vercel/lib/collector.js new file mode 100644 index 0000000..054eba4 --- /dev/null +++ b/tools/biologix-public-intel/vercel/lib/collector.js @@ -0,0 +1,620 @@ +import { + BIOLOGIX_BASE_URL, + DEEP_SCAN_INTERVAL_MS, + POLL_INTERVAL_MINUTES, + RETENTION_DAYS, + aggregateReport, + buildObservations, + clusterProbableBaskets, + compactObservationMap, + detectPublicTechnology, + detectTrackers, + diffObservations, + inventorySummary, + parseSitemapIndex, + publicAggregateProbes, + relevantNamespaces, + safeDurationSince, + summarizeUrlset, +} from "./biologix-intel-core.js"; + +const STORE_NAME = "biologix-production"; +const REQUEST_TIMEOUT_MS = 18_000; +const MAX_TEXT_BYTES = 2 * 1024 * 1024; +const MAX_DAILY_EVENTS = 300; +const MAX_SITEMAPS = 20; +const API_PREFIX = "/api/biologix-intel"; + +function jsonResponse(payload, status = 200, extraHeaders = {}) { + return new Response(JSON.stringify(payload, null, 2), { + status, + headers: { + "Cache-Control": "no-store", + "Content-Type": "application/json; charset=utf-8", + "Referrer-Policy": "no-referrer", + "X-Content-Type-Options": "nosniff", + "X-Robots-Tag": "noindex, nofollow, noarchive", + ...extraHeaders, + }, + }); +} + +function utcDateKey(iso) { + return iso.slice(0, 10); +} + +function clampReportHours(value) { + const parsed = Number.parseInt(value ?? "24", 10); + if (!Number.isFinite(parsed)) return 24; + return Math.min(RETENTION_DAYS * 24, Math.max(1, parsed)); +} + +function publicHeaders(response) { + const headerNames = [ + "cache-control", + "cf-cache-status", + "last-modified", + "platform", + "server", + "x-litespeed-cache", + "x-powered-by", + "x-turbo-charged-by", + ]; + return Object.fromEntries( + headerNames + .map((name) => [name, response.headers.get(name)]) + .filter(([, value]) => value !== null), + ); +} + +async function fetchBounded(url, options = {}) { + const started = Date.now(); + const response = await fetch(url, { + method: "GET", + redirect: "follow", + headers: { + Accept: options.accept ?? "application/json,text/html;q=0.9,*/*;q=0.1", + "User-Agent": + "OVO-Public-Intelligence/1.0 (+low-frequency aggregate research)", + }, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + const buffer = await response.arrayBuffer(); + if (buffer.byteLength > (options.maxBytes ?? MAX_TEXT_BYTES)) { + throw new Error(`Response exceeded byte limit: ${url}`); + } + return { + response, + text: new TextDecoder().decode(buffer), + bytes: buffer.byteLength, + duration_ms: Date.now() - started, + }; +} + +function withQuery(url, values) { + const parsed = new URL(url); + for (const [key, value] of Object.entries(values)) { + parsed.searchParams.set(key, String(value)); + } + return parsed.toString(); +} + +async function fetchJsonCollection(url) { + const first = await fetchBounded(withQuery(url, { page: 1 }), { + accept: "application/json", + maxBytes: 8 * 1024 * 1024, + }); + if (!first.response.ok) { + throw new Error(`GET ${url} returned ${first.response.status}`); + } + const firstRecords = JSON.parse(first.text); + if (!Array.isArray(firstRecords)) { + throw new Error(`Expected JSON list from ${url}`); + } + const totalPages = Math.min( + 20, + Number.parseInt(first.response.headers.get("x-wp-totalpages") ?? "1", 10), + ); + const remaining = + totalPages > 1 + ? await Promise.all( + Array.from({ length: totalPages - 1 }, async (_, index) => { + const page = index + 2; + const result = await fetchBounded(withQuery(url, { page }), { + accept: "application/json", + maxBytes: 8 * 1024 * 1024, + }); + if (!result.response.ok) { + throw new Error(`GET ${url} page ${page} returned ${result.response.status}`); + } + const records = JSON.parse(result.text); + if (!Array.isArray(records)) { + throw new Error(`Expected JSON list from ${url} page ${page}`); + } + return records; + }), + ) + : []; + return { + records: [...firstRecords, ...remaining.flat()], + meta: { + status: first.response.status, + total_pages: totalPages, + total_items: Number.parseInt( + first.response.headers.get("x-wp-total") ?? String(firstRecords.length), + 10, + ), + first_page_duration_ms: first.duration_ms, + }, + }; +} + +async function quickHomepageProbe() { + const result = await fetchBounded(`${BIOLOGIX_BASE_URL}/`, { + accept: "text/html", + maxBytes: 3 * 1024 * 1024, + }); + return { + status: result.response.status, + duration_ms: result.duration_ms, + bytes: result.bytes, + headers: publicHeaders(result.response), + trackers: detectTrackers(result.text), + technology: detectPublicTechnology( + result.text, + Object.fromEntries( + [...result.response.headers].map(([key, value]) => [key, value]), + ), + ), + html: result.text, + }; +} + +function publicRouteStatus(response) { + if (response.status === 401 || response.status === 403) return "authenticated"; + if (response.status === 404) return "not_found"; + if (response.ok) return "public"; + return `http_${response.status}`; +} + +async function probeAggregateEndpoint(name, path) { + try { + const result = await fetchBounded(`${BIOLOGIX_BASE_URL}${path}`, { + accept: "application/json", + maxBytes: 256 * 1024, + }); + return { + name, + path, + status_code: result.response.status, + visibility: publicRouteStatus(result.response), + }; + } catch (error) { + return { name, path, status_code: null, visibility: "error", error: error.message }; + } +} + +async function dnsQuery(type) { + const url = new URL("https://cloudflare-dns.com/dns-query"); + url.searchParams.set("name", new URL(BIOLOGIX_BASE_URL).hostname); + url.searchParams.set("type", type); + const result = await fetchBounded(url.toString(), { + accept: "application/dns-json", + maxBytes: 256 * 1024, + }); + const payload = JSON.parse(result.text); + return { + type, + status: payload.Status, + answers: (payload.Answer ?? []).map((answer) => ({ + name: answer.name, + ttl: answer.TTL, + data: answer.data, + })), + }; +} + +async function collectSitemapSignals(robotsText) { + const sitemapFromRobots = robotsText.match(/^Sitemap:\s*(\S+)/im)?.[1]; + const candidates = [ + sitemapFromRobots, + `${BIOLOGIX_BASE_URL}/sitemap_index.xml`, + `${BIOLOGIX_BASE_URL}/wp-sitemap.xml`, + ].filter(Boolean); + let indexResult = null; + for (const candidate of [...new Set(candidates)]) { + try { + const result = await fetchBounded(candidate, { + accept: "application/xml,text/xml", + maxBytes: 2 * 1024 * 1024, + }); + if (result.response.ok && result.text.includes(" { + try { + const result = await fetchBounded(entry.location, { + accept: "application/xml,text/xml", + maxBytes: 4 * 1024 * 1024, + }); + const summary = summarizeUrlset(result.text); + return { + name: new URL(entry.location).pathname.split("/").at(-1), + status: result.response.status, + ...summary, + }; + } catch (error) { + return { + name: new URL(entry.location).pathname.split("/").at(-1), + status: null, + url_count: 0, + error: error.message, + }; + } + }), + ); + return { + available: true, + index_name: new URL(indexResult.url).pathname.split("/").at(-1), + index_duration_ms: indexResult.duration_ms, + sitemap_count: childResults.length, + url_count: childResults.reduce((total, child) => total + child.url_count, 0), + sitemaps: childResults, + }; +} + +async function collectDeepSignals(homepage) { + const [robotsResult, wpRootResult, dnsResults, aggregateEndpoints] = + await Promise.all([ + fetchBounded(`${BIOLOGIX_BASE_URL}/robots.txt`, { + accept: "text/plain", + maxBytes: 256 * 1024, + }), + fetchBounded(`${BIOLOGIX_BASE_URL}/wp-json/`, { + accept: "application/json", + maxBytes: 8 * 1024 * 1024, + }), + Promise.all(["A", "AAAA", "NS", "MX"].map(dnsQuery)), + Promise.all(publicAggregateProbes().map(([name, path]) => + probeAggregateEndpoint(name, path), + )), + ]); + const wpRoot = JSON.parse(wpRootResult.text); + let jetpack = null; + try { + const result = await fetchBounded( + `${BIOLOGIX_BASE_URL}/wp-json/jetpack/v4/connection`, + { accept: "application/json", maxBytes: 128 * 1024 }, + ); + if (result.response.ok) { + const payload = JSON.parse(result.text); + jetpack = { + installed: true, + active: Boolean(payload.isActive), + registered: Boolean(payload.isRegistered), + user_connected: Boolean(payload.isUserConnected), + site_marked_public: Boolean(payload.isPublic), + }; + } + } catch { + jetpack = { installed: true, visibility: "unavailable" }; + } + + return { + captured_at: new Date().toISOString(), + robots: { + status: robotsResult.response.status, + bytes: robotsResult.bytes, + sitemap_declared: /^Sitemap:\s*(\S+)/im.test(robotsResult.text), + }, + sitemap: await collectSitemapSignals(robotsResult.text), + wordpress: { + name: String(wpRoot.name ?? ""), + description: String(wpRoot.description ?? ""), + namespace_count: Array.isArray(wpRoot.namespaces) + ? wpRoot.namespaces.length + : 0, + route_count: + wpRoot.routes && typeof wpRoot.routes === "object" + ? Object.keys(wpRoot.routes).length + : 0, + relevant_namespaces: relevantNamespaces(wpRoot.namespaces ?? []), + }, + analytics_and_sales_endpoints: aggregateEndpoints, + jetpack, + dns: dnsResults, + technology: homepage.technology, + trackers: homepage.trackers, + traffic_truth: { + visitor_counts_publicly_available: false, + reason: + "Installed tools and tags can be detected publicly, but their visitor and revenue reports require authorization.", + }, + }; +} + +export async function collectPublicSnapshot(previousState, trigger) { + const started = Date.now(); + const parentUrl = `${BIOLOGIX_BASE_URL}/wp-json/wc/store/v1/products?per_page=100&orderby=popularity&order=desc`; + const variationUrl = `${BIOLOGIX_BASE_URL}/wp-json/wc/store/v1/products?per_page=100&type=variation`; + const wpProductUrl = `${BIOLOGIX_BASE_URL}/wp-json/wp/v2/product?per_page=100&_fields=id,modified_gmt`; + const [parents, variations, wpProducts, homepage] = await Promise.all([ + fetchJsonCollection(parentUrl), + fetchJsonCollection(variationUrl), + fetchJsonCollection(wpProductUrl), + quickHomepageProbe(), + ]); + + const capturedAt = new Date().toISOString(); + const observations = buildObservations( + parents.records, + variations.records, + wpProducts.records, + ); + const inventory = inventorySummary(observations); + const events = diffObservations( + previousState?.latest_observations ?? {}, + observations, + capturedAt, + ); + const baskets = clusterProbableBaskets( + events, + previousState?.latest_snapshot?.captured_at, + capturedAt, + ); + const shouldDeepScan = + safeDurationSince(previousState?.public_site_signals?.captured_at) === null || + safeDurationSince(previousState?.public_site_signals?.captured_at) >= + DEEP_SCAN_INTERVAL_MS; + let deepSignals = previousState?.public_site_signals ?? null; + let deepScanError = null; + if (shouldDeepScan) { + try { + deepSignals = await collectDeepSignals(homepage); + } catch (error) { + deepScanError = error.message; + } + } + + return { + summary: { + captured_at: capturedAt, + trigger, + duration_ms: Date.now() - started, + parent_count: parents.records.length, + variation_count: variations.records.length, + ...inventory, + homepage: { + status: homepage.status, + duration_ms: homepage.duration_ms, + bytes: homepage.bytes, + headers: homepage.headers, + trackers: homepage.trackers, + }, + event_count: events.length, + probable_basket_count: baskets.length, + deep_scan_performed: shouldDeepScan && deepScanError === null, + deep_scan_error: deepScanError, + }, + observations, + events, + baskets, + publicSiteSignals: deepSignals, + }; +} + +export function newState() { + return { + version: 1, + day_keys: [], + latest_snapshot: null, + latest_observations: {}, + public_site_signals: null, + last_attempt_at: null, + last_success_at: null, + last_error_at: null, + last_error: null, + successful_runs: 0, + failed_runs: 0, + }; +} + +class LegacyBiologixIntelStore { + constructor(ctx, env) { + this.ctx = ctx; + this.env = env; + } + + async runSnapshot(trigger = "cron") { + const state = (await this.ctx.storage.get("state")) ?? newState(); + const lastAttemptAge = safeDurationSince(state.last_attempt_at); + if (trigger === "cron" && lastAttemptAge !== null && lastAttemptAge < 4 * 60_000) { + return { skipped: true, reason: "duplicate_trigger_guard", ...this.health(state) }; + } + state.last_attempt_at = new Date().toISOString(); + await this.ctx.storage.put("state", state); + + try { + const result = await collectPublicSnapshot(state, trigger); + const dateKey = utcDateKey(result.summary.captured_at); + const storageKey = `day:${dateKey}`; + const dayRecord = (await this.ctx.storage.get(storageKey)) ?? { + date: dateKey, + snapshots: [], + events: [], + baskets: [], + events_truncated: 0, + }; + dayRecord.snapshots.push(result.summary); + dayRecord.events.push(...result.events); + dayRecord.baskets.push(...result.baskets); + if (dayRecord.events.length > MAX_DAILY_EVENTS) { + const overflow = dayRecord.events.length - MAX_DAILY_EVENTS; + dayRecord.events.splice(0, overflow); + dayRecord.events_truncated += overflow; + } + + if (!state.day_keys.includes(dateKey)) state.day_keys.push(dateKey); + state.day_keys.sort(); + const expiredKeys = state.day_keys.splice( + 0, + Math.max(0, state.day_keys.length - RETENTION_DAYS), + ); + state.latest_snapshot = result.summary; + state.latest_observations = compactObservationMap(result.observations); + state.public_site_signals = result.publicSiteSignals; + state.last_success_at = result.summary.captured_at; + state.last_error = null; + state.successful_runs += 1; + + await this.ctx.storage.put({ + state, + [storageKey]: dayRecord, + }); + if (expiredKeys.length) { + await this.ctx.storage.delete(expiredKeys.map((key) => `day:${key}`)); + } + return { + skipped: false, + snapshot: result.summary, + probable_baskets: result.baskets, + }; + } catch (error) { + state.last_error_at = new Date().toISOString(); + state.last_error = error instanceof Error ? error.message : String(error); + state.failed_runs += 1; + await this.ctx.storage.put("state", state); + throw error; + } + } + + health(state) { + const now = Date.now(); + const lastSuccessAge = safeDurationSince(state.last_success_at, now); + const healthy = + lastSuccessAge !== null && + lastSuccessAge <= (POLL_INTERVAL_MINUTES + 10) * 60_000 && + !state.last_error; + return { + status: + state.last_success_at === null ? "awaiting_first_run" : healthy ? "healthy" : "degraded", + cadence_minutes: POLL_INTERVAL_MINUTES, + last_attempt_at: state.last_attempt_at, + last_success_at: state.last_success_at, + last_success_age_seconds: + lastSuccessAge === null ? null : Math.floor(lastSuccessAge / 1000), + last_error_at: state.last_error_at, + last_error: state.last_error, + successful_runs: state.successful_runs, + failed_runs: state.failed_runs, + retention_days: RETENTION_DAYS, + collector: "public_get_only_no_customer_data", + }; + } + + async getHealth() { + const state = (await this.ctx.storage.get("state")) ?? newState(); + return this.health(state); + } + + async getLatest() { + const state = (await this.ctx.storage.get("state")) ?? newState(); + return { + health: this.health(state), + snapshot: state.latest_snapshot, + public_site_signals: state.public_site_signals, + }; + } + + async getReport(hours = 24) { + const state = (await this.ctx.storage.get("state")) ?? newState(); + const since = new Date(Date.now() - clampReportHours(hours) * 60 * 60 * 1000); + const dayKeys = state.day_keys.filter( + (key) => Date.parse(`${key}T23:59:59.999Z`) >= since.getTime(), + ); + const values = await this.ctx.storage.get( + dayKeys.map((key) => `day:${key}`), + ); + const days = dayKeys.map((key) => values.get(`day:${key}`)).filter(Boolean); + return { + health: this.health(state), + ...aggregateReport(days, state, since.toISOString()), + }; + } +} + +function getStore(env) { + if (!env.BIOLOGIX_INTEL) { + throw new Error("BIOLOGIX_INTEL Durable Object binding is unavailable"); + } + const id = env.BIOLOGIX_INTEL.idFromName(STORE_NAME); + return env.BIOLOGIX_INTEL.get(id); +} + +function isAuthorized(request, env) { + const expected = env.BIOLOGIX_INTEL_TOKEN; + if (!expected) return false; + return request.headers.get("Authorization") === `Bearer ${expected}`; +} + +export async function handleBiologixIntelRequest(request, env) { + const url = new URL(request.url); + const route = url.pathname.slice(API_PREFIX.length) || "/"; + const store = getStore(env); + + if (request.method === "GET" && route === "/health") { + return jsonResponse(await store.getHealth()); + } + + if (!isAuthorized(request, env)) { + return jsonResponse( + { + error: "unauthorized", + message: "Use the private Biologix intelligence bearer token.", + }, + 401, + { "WWW-Authenticate": 'Bearer realm="biologix-intel"' }, + ); + } + + if (request.method === "GET" && route === "/latest") { + return jsonResponse(await store.getLatest()); + } + if (request.method === "GET" && route === "/report") { + return jsonResponse(await store.getReport(clampReportHours(url.searchParams.get("hours")))); + } + if (request.method === "POST" && route === "/snapshot") { + try { + return jsonResponse(await store.runSnapshot("manual"), 201); + } catch (error) { + return jsonResponse( + { + error: "snapshot_failed", + message: error instanceof Error ? error.message : String(error), + }, + 502, + ); + } + } + return jsonResponse({ error: "not_found" }, 404); +} + +export async function runScheduledBiologixSnapshot(env) { + const result = await getStore(env).runSnapshot("cron"); + console.log( + JSON.stringify({ + event: "biologix_public_intel_snapshot", + ...result, + }), + ); + return result; +} diff --git a/tools/biologix-public-intel/vercel/lib/http.js b/tools/biologix-public-intel/vercel/lib/http.js new file mode 100644 index 0000000..66c6f47 --- /dev/null +++ b/tools/biologix-public-intel/vercel/lib/http.js @@ -0,0 +1,37 @@ +import { timingSafeEqual } from "node:crypto"; + +export function json(payload, status = 200, headers = {}) { + return new Response(JSON.stringify(payload, null, 2), { + status, + headers: { + "Cache-Control": "no-store", + "Content-Type": "application/json; charset=utf-8", + "Referrer-Policy": "no-referrer", + "X-Content-Type-Options": "nosniff", + "X-Robots-Tag": "noindex, nofollow, noarchive", + ...headers, + }, + }); +} + +export function hasBearerToken(request, expected) { + if (!expected) return false; + const actual = + (typeof request.headers?.get === "function" + ? request.headers.get("Authorization") + : request.headers?.authorization) ?? ""; + const wanted = `Bearer ${expected}`; + if (actual.length !== wanted.length) return false; + return timingSafeEqual(Buffer.from(actual), Buffer.from(wanted)); +} + +export function unauthorized() { + return json( + { + error: "unauthorized", + message: "Use the private Biologix intelligence bearer token.", + }, + 401, + { "WWW-Authenticate": 'Bearer realm="biologix-intel"' }, + ); +} diff --git a/tools/biologix-public-intel/vercel/lib/state.js b/tools/biologix-public-intel/vercel/lib/state.js new file mode 100644 index 0000000..36af40d --- /dev/null +++ b/tools/biologix-public-intel/vercel/lib/state.js @@ -0,0 +1,128 @@ +import { + POLL_INTERVAL_MINUTES, + RETENTION_DAYS, + aggregateReport, + compactObservationMap, + safeDurationSince, +} from "./biologix-intel-core.js"; + +const MAX_DAILY_EVENTS = 300; + +export function createState() { + return { + version: 1, + day_keys: [], + latest_snapshot: null, + latest_observations: {}, + public_site_signals: null, + last_attempt_at: null, + last_success_at: null, + last_error_at: null, + last_error: null, + successful_runs: 0, + failed_runs: 0, + }; +} + +export function createDay(date) { + return { + date, + snapshots: [], + events: [], + baskets: [], + events_truncated: 0, + }; +} + +export function healthForState(state, now = Date.now()) { + const lastSuccessAge = safeDurationSince(state.last_success_at, now); + const healthy = + lastSuccessAge !== null && + lastSuccessAge <= (POLL_INTERVAL_MINUTES + 10) * 60_000 && + !state.last_error; + + return { + status: + state.last_success_at === null + ? "awaiting_first_run" + : healthy + ? "healthy" + : "degraded", + cadence_minutes: POLL_INTERVAL_MINUTES, + last_attempt_at: state.last_attempt_at, + last_success_at: state.last_success_at, + last_success_age_seconds: + lastSuccessAge === null ? null : Math.floor(lastSuccessAge / 1000), + last_error_at: state.last_error_at, + last_error: state.last_error, + successful_runs: state.successful_runs, + failed_runs: state.failed_runs, + retention_days: RETENTION_DAYS, + collector: "public_get_only_no_customer_data", + infrastructure: "vercel_cron_private_blob", + }; +} + +export function shouldSkipCron(state, now = Date.now()) { + const age = safeDurationSince(state.last_attempt_at, now); + return age !== null && age < 4 * 60_000; +} + +export function markAttempt(state, attemptedAt) { + return { + ...state, + last_attempt_at: attemptedAt, + }; +} + +export function applySnapshot(state, day, result) { + const nextDay = structuredClone(day); + nextDay.snapshots.push(result.summary); + nextDay.events.push(...result.events); + nextDay.baskets.push(...result.baskets); + if (nextDay.events.length > MAX_DAILY_EVENTS) { + const overflow = nextDay.events.length - MAX_DAILY_EVENTS; + nextDay.events.splice(0, overflow); + nextDay.events_truncated += overflow; + } + + const dateKey = result.summary.captured_at.slice(0, 10); + const dayKeys = [...new Set([...state.day_keys, dateKey])].sort(); + const expiredDayKeys = dayKeys.splice( + 0, + Math.max(0, dayKeys.length - RETENTION_DAYS), + ); + const nextState = { + ...state, + day_keys: dayKeys, + latest_snapshot: result.summary, + latest_observations: compactObservationMap(result.observations), + public_site_signals: result.publicSiteSignals, + last_success_at: result.summary.captured_at, + last_error: null, + successful_runs: state.successful_runs + 1, + }; + + return { state: nextState, day: nextDay, expiredDayKeys }; +} + +export function markFailure(state, error, failedAt) { + return { + ...state, + last_error_at: failedAt, + last_error: error instanceof Error ? error.message : String(error), + failed_runs: state.failed_runs + 1, + }; +} + +export function reportFromDays(days, state, hours, now = Date.now()) { + const parsed = Number.parseInt(hours ?? "24", 10); + const clampedHours = Number.isFinite(parsed) + ? Math.min(RETENTION_DAYS * 24, Math.max(1, parsed)) + : 24; + const since = new Date(now - clampedHours * 60 * 60 * 1000); + return { + health: healthForState(state, now), + ...aggregateReport(days, state, since.toISOString()), + }; +} diff --git a/tools/biologix-public-intel/vercel/package-lock.json b/tools/biologix-public-intel/vercel/package-lock.json new file mode 100644 index 0000000..abeeac3 --- /dev/null +++ b/tools/biologix-public-intel/vercel/package-lock.json @@ -0,0 +1,369 @@ +{ + "name": "biologix-public-intel-vercel", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "biologix-public-intel-vercel", + "version": "1.0.0", + "dependencies": { + "@vercel/blob": "2.6.1" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@vercel/blob": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/@vercel/blob/-/blob-2.6.1.tgz", + "integrity": "sha512-KTJytw85j1XQBxjN5d6UXI7fIWNQe1jotn4nWN+0hePqLs+Qi1B3jHdQcSKFGF0m2rsy9uhPT6GOXMtHe3qNzg==", + "license": "Apache-2.0", + "dependencies": { + "@vercel/oidc": "^3.6.1", + "async-retry": "^1.3.3", + "is-buffer": "^2.0.5", + "is-node-process": "^1.2.0", + "throttleit": "^2.1.0", + "undici": "^6.23.0" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@vercel/cli-config": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/@vercel/cli-config/-/cli-config-0.2.1.tgz", + "integrity": "sha512-RhfyXmRLHdbnry8RJqHDc+5rGxMZ0bu+fpysZjtv3bE+BubpuwxTancHOKiH5zKQREsdwFVr3mOI2kOvxlOyxA==", + "license": "Apache-2.0", + "dependencies": { + "xdg-app-paths": "5", + "zod": "4.1.11" + } + }, + "node_modules/@vercel/cli-exec": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@vercel/cli-exec/-/cli-exec-1.0.0.tgz", + "integrity": "sha512-kQF8LGie/Hbdq9/psJxLE7owRTcqMQMhgybU04gCeR7cbQAr5t8OrjefDNColJv1QSSucFt4pLwRiARVmlOnug==", + "license": "Apache-2.0", + "dependencies": { + "execa": "5.1.1" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@vercel/oidc": { + "version": "3.8.1", + "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.8.1.tgz", + "integrity": "sha512-ufdalm2MWOYksyj8KVpWjoOFPJO6zoYpuyvIggIQ2bB0CFCjTCiTkGXHqAKwG77GVRjOaN3/8S5ITlZpXWmqOw==", + "license": "Apache-2.0", + "dependencies": { + "@vercel/cli-config": "0.2.1", + "@vercel/cli-exec": "1.0.0", + "jose": "^5.9.6" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/async-retry": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", + "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", + "license": "MIT", + "dependencies": { + "retry": "0.13.1" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/execa": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", + "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", + "license": "MIT", + "dependencies": { + "cross-spawn": "^7.0.3", + "get-stream": "^6.0.0", + "human-signals": "^2.1.0", + "is-stream": "^2.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^4.0.1", + "onetime": "^5.1.2", + "signal-exit": "^3.0.3", + "strip-final-newline": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "node_modules/get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/human-signals": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", + "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", + "license": "Apache-2.0", + "engines": { + "node": ">=10.17.0" + } + }, + "node_modules/is-buffer": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", + "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/is-node-process": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-node-process/-/is-node-process-1.2.0.tgz", + "integrity": "sha512-Vg4o6/fqPxIjtxgUH5QLJhwZ7gW5diGCVlXpuUfELC62CuxM1iHcRe51f2W1FDy04Ai4KJkagKjx3XaqyfRKXw==", + "license": "MIT" + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/jose": { + "version": "5.10.0", + "resolved": "https://registry.npmjs.org/jose/-/jose-5.10.0.tgz", + "integrity": "sha512-s+3Al/p9g32Iq+oqXxkW//7jk2Vig6FF1CFqzVXoTUXt2qz89YWbL+OwS17NFYEvxC35n0FKeGO2LGYSxeM2Gg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, + "node_modules/merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "license": "MIT" + }, + "node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/npm-run-path": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", + "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", + "license": "MIT", + "dependencies": { + "path-key": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "license": "MIT", + "dependencies": { + "mimic-fn": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/os-paths": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/os-paths/-/os-paths-4.4.0.tgz", + "integrity": "sha512-wrAwOeXp1RRMFfQY8Sy7VaGVmPocaLwSFOYCGKSyo8qmJ+/yaafCl5BCA1IQZWqFSRBrKDYFeR9d/VyQzfH/jg==", + "license": "MIT", + "engines": { + "node": ">= 6.0" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "license": "ISC" + }, + "node_modules/strip-final-newline": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", + "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/throttleit": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/throttleit/-/throttleit-2.1.0.tgz", + "integrity": "sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/undici": { + "version": "6.28.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.28.0.tgz", + "integrity": "sha512-LIY910g9TI13YS95lrMFrs8Rm/u/irgHeTWoKCoteeJ04CUJ92eEfj0rVn+7VKMPBpUPiUoBKfhNyLI23EE/KA==", + "license": "MIT", + "engines": { + "node": ">=18.17" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/xdg-app-paths": { + "version": "5.5.1", + "resolved": "https://registry.npmjs.org/xdg-app-paths/-/xdg-app-paths-5.5.1.tgz", + "integrity": "sha512-hI3flOB4PLZIy5prbtTpirobtPE2ZtZ52szO+2mM9Efp6ErM398La+C1lIpNWDfNoQk+6Lsi6nMcCwVB7pxeMQ==", + "license": "MIT", + "dependencies": { + "os-paths": "^4.0.1", + "xdg-portable": "^7.2.0" + }, + "engines": { + "node": ">= 6.0" + } + }, + "node_modules/xdg-portable": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/xdg-portable/-/xdg-portable-7.3.0.tgz", + "integrity": "sha512-sqMMuL1rc0FmMBOzCpd0yuy9trqF2yTTVe+E9ogwCSWQCdDEtQUwrZPT6AxqtsFGRNxycgncbP/xmOOSPw5ZUw==", + "license": "MIT", + "dependencies": { + "os-paths": "^4.0.1" + }, + "engines": { + "node": ">= 6.0" + } + }, + "node_modules/zod": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.1.11.tgz", + "integrity": "sha512-WPsqwxITS2tzx1bzhIKsEs19ABD5vmCVa4xBo2tq/SrV4RNZtfws1EnCWQXM6yh8bD08a1idvkB5MZSBiZsjwg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/tools/biologix-public-intel/vercel/package.json b/tools/biologix-public-intel/vercel/package.json new file mode 100644 index 0000000..156c1ef --- /dev/null +++ b/tools/biologix-public-intel/vercel/package.json @@ -0,0 +1,16 @@ +{ + "name": "biologix-public-intel-vercel", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "test": "node --test test/*.test.mjs", + "check": "node --check api/cron.js && node --check 'api/biologix-intel/[route].js' && node --check lib/*.js" + }, + "dependencies": { + "@vercel/blob": "2.6.1" + }, + "engines": { + "node": ">=20" + } +} diff --git a/tools/biologix-public-intel/vercel/test/biologix-intel.test.mjs b/tools/biologix-public-intel/vercel/test/biologix-intel.test.mjs new file mode 100644 index 0000000..b4ec825 --- /dev/null +++ b/tools/biologix-public-intel/vercel/test/biologix-intel.test.mjs @@ -0,0 +1,168 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + aggregateReport, + buildObservations, + clusterProbableBaskets, + detectPublicTechnology, + detectTrackers, + diffObservations, + inventorySummary, + parseSitemapIndex, + parseStockQuantity, + summarizeUrlset, +} from "../lib/biologix-intel-core.js"; + +function product(overrides = {}) { + return { + id: 1, + parent: 0, + name: "Example", + type: "simple", + variation: "", + sku: "SKU-1", + prices: { price: "1000", regular_price: "1000", sale_price: "1000" }, + stock_availability: { text: "10 in stock" }, + is_in_stock: true, + is_on_backorder: false, + is_purchasable: true, + permalink: "https://example.com/product/example", + ...overrides, + }; +} + +test("stock parsing distinguishes exact, unknown, and out of stock", () => { + assert.equal(parseStockQuantity("17 in stock (can be backordered)", true), 17); + assert.equal(parseStockQuantity("In stock", true), null); + assert.equal(parseStockQuantity("Out of stock", false), 0); +}); + +test("variable parent stock is not double-counted when children are exact", () => { + const parent = product({ + id: 10, + type: "variable", + stock_availability: { text: "20 in stock" }, + }); + const variation = product({ + id: 11, + parent: 10, + type: "variation", + variation: "Amount: 10mg", + stock_availability: { text: "7 in stock" }, + }); + const observations = buildObservations([parent], [variation], []); + const summary = inventorySummary(observations); + assert.equal(summary.exact_inventory_units, 7); + assert.equal(summary.displayed_inventory_value_cents, 7000); + assert.equal(observations.find((item) => item.key === "product:10").track_inventory, false); +}); + +test("inventory deltas and synchronized timestamps form an inferred basket", () => { + const previous = buildObservations([product()], [], [ + { id: 1, modified_gmt: "2026-07-25T00:00:00Z" }, + ]); + const current = buildObservations( + [ + product({ stock_availability: { text: "9 in stock" } }), + product({ + id: 2, + sku: "SKU-2", + name: "Second", + prices: { price: "2000" }, + stock_availability: { text: "4 in stock" }, + }), + ], + [], + [ + { id: 1, modified_gmt: "2026-07-25T00:05:01Z" }, + { id: 2, modified_gmt: "2026-07-25T00:05:04Z" }, + ], + ); + const previousMap = Object.fromEntries(previous.map((item) => [item.key, item])); + previousMap["product:2"] = { + ...current.find((item) => item.key === "product:2"), + stock_quantity: 5, + }; + const events = diffObservations( + previousMap, + current, + "2026-07-25T00:05:10Z", + ); + const baskets = clusterProbableBaskets( + events, + "2026-07-25T00:00:00Z", + "2026-07-25T00:05:10Z", + ); + assert.equal(events.filter((event) => event.event_type === "inventory_decrease").length, 2); + assert.equal(baskets.length, 1); + assert.equal(baskets[0].unit_count, 2); + assert.equal(baskets[0].displayed_value_cents, 3000); +}); + +test("tracker and public technology detection is deterministic", () => { + const html = ` + + + + + `; + assert.deepEqual(detectTrackers(html), [ + { provider: "google_analytics", public_id: "G-ABCDEF12" }, + { provider: "meta_pixel", public_id: "123456789" }, + ]); + assert.deepEqual( + detectPublicTechnology(html, { server: "cloudflare", platform: "hostinger" }), + { + technologies: ["cloudflare", "hostinger", "woocommerce", "wordpress"], + plugins: ["woocommerce"], + themes: ["woostify"], + }, + ); +}); + +test("sitemap parsers count public pages without retaining page contents", () => { + const index = ` + + https://example.com/a.xml2026-01-01 + https://example.com/b.xml + + `; + assert.equal(parseSitemapIndex(index).length, 2); + const urlset = ` + + https://example.com/a2026-01-01 + https://example.com/b2026-02-01 + + `; + assert.deepEqual(summarizeUrlset(urlset), { + url_count: 2, + latest_lastmod: "2026-02-01", + }); +}); + +test("reports retain evidence boundaries and never call inventory a paid sale", () => { + const state = { + latest_snapshot: { captured_at: "2026-07-25T01:00:00Z" }, + public_site_signals: { traffic_truth: { visitor_counts_publicly_available: false } }, + }; + const days = [ + { + snapshots: [{ captured_at: "2026-07-25T01:00:00Z" }], + events: [ + { + observed_at: "2026-07-25T01:00:00Z", + event_type: "inventory_decrease", + quantity_delta: -2, + displayed_value_cents: 5000, + }, + ], + baskets: [], + }, + ]; + const report = aggregateReport(days, state, "2026-07-25T00:00:00Z"); + assert.equal(report.movement.observed_units_down, 2); + assert.equal(report.movement.displayed_price_gmv_signal_cents, 5000); + assert.equal(report.traffic.direct_visitor_counts_available, false); + assert.match(report.evidence_boundary.unavailable, /cannot prove payment/); +}); diff --git a/tools/biologix-public-intel/vercel/test/state.test.mjs b/tools/biologix-public-intel/vercel/test/state.test.mjs new file mode 100644 index 0000000..5772561 --- /dev/null +++ b/tools/biologix-public-intel/vercel/test/state.test.mjs @@ -0,0 +1,66 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { normalizeBlobEtag } from "../lib/blob-store.js"; +import { + applySnapshot, + createDay, + createState, + healthForState, + shouldSkipCron, +} from "../lib/state.js"; +import { hasBearerToken } from "../lib/http.js"; + +test("health becomes current after a successful snapshot", () => { + const state = createState(); + const capturedAt = "2026-07-25T01:00:00.000Z"; + const applied = applySnapshot(state, createDay("2026-07-25"), { + summary: { captured_at: capturedAt, trigger: "cron" }, + observations: [], + events: [], + baskets: [], + publicSiteSignals: null, + }); + const health = healthForState( + applied.state, + Date.parse("2026-07-25T01:01:00.000Z"), + ); + assert.equal(health.status, "healthy"); + assert.equal(health.successful_runs, 1); + assert.equal(health.infrastructure, "vercel_cron_private_blob"); +}); + +test("cron duplicate guard only skips recent attempts", () => { + const recent = { + ...createState(), + last_attempt_at: "2026-07-25T01:00:00.000Z", + }; + assert.equal( + shouldSkipCron(recent, Date.parse("2026-07-25T01:03:00.000Z")), + true, + ); + assert.equal( + shouldSkipCron(recent, Date.parse("2026-07-25T01:05:00.000Z")), + false, + ); +}); + +test("bearer authentication accepts Vercel's Node request headers", () => { + assert.equal( + hasBearerToken({ headers: { authorization: "Bearer secret" } }, "secret"), + true, + ); + assert.equal( + hasBearerToken({ headers: { authorization: "Bearer wrong" } }, "secret"), + false, + ); +}); + +test("weak Blob read ETags normalize for conditional writes", () => { + assert.equal( + normalizeBlobEtag('W/"d41cd3df8fc9aba147758c3dd2d42c1c"'), + '"d41cd3df8fc9aba147758c3dd2d42c1c"', + ); + assert.equal(normalizeBlobEtag('"strong"'), '"strong"'); + assert.equal(normalizeBlobEtag(null), null); +}); diff --git a/tools/biologix-public-intel/vercel/vercel.json b/tools/biologix-public-intel/vercel/vercel.json new file mode 100644 index 0000000..6ddb635 --- /dev/null +++ b/tools/biologix-public-intel/vercel/vercel.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://openapi.vercel.sh/vercel.json", + "regions": [ + "sfo1" + ], + "crons": [ + { + "path": "/api/cron", + "schedule": "*/15 * * * *" + } + ], + "functions": { + "api/**/*.js": { + "maxDuration": 300 + } + } +} From 367cd8f6cad8f4dc0265d8a41da8bec9266b7518 Mon Sep 17 00:00:00 2001 From: Alex Weinstein Date: Fri, 24 Jul 2026 19:03:17 -0700 Subject: [PATCH 5/6] fix: bypass cache for Biologix state reads --- tools/biologix-public-intel/vercel/lib/blob-store.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/biologix-public-intel/vercel/lib/blob-store.js b/tools/biologix-public-intel/vercel/lib/blob-store.js index 6b636cd..603414b 100644 --- a/tools/biologix-public-intel/vercel/lib/blob-store.js +++ b/tools/biologix-public-intel/vercel/lib/blob-store.js @@ -28,7 +28,10 @@ function dayPath(dateKey) { } async function readJson(pathname) { - const result = await get(pathname, { access: "private" }); + const result = await get(pathname, { + access: "private", + useCache: false, + }); if (!result) return { value: null, etag: null }; if (result.statusCode !== 200 || !result.stream) { throw new Error(`Blob read failed for ${pathname}: ${result.statusCode}`); From f3c2ba53ad92eb7e4bde227a0efc915afc61d15b Mon Sep 17 00:00:00 2001 From: Alex Weinstein Date: Sat, 25 Jul 2026 10:33:19 -0700 Subject: [PATCH 6/6] fix: bypass storefront cache during intelligence polls --- .../vercel/lib/collector.js | 28 +++++++++++++++++-- .../vercel/test/biologix-intel.test.mjs | 13 +++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/tools/biologix-public-intel/vercel/lib/collector.js b/tools/biologix-public-intel/vercel/lib/collector.js index 054eba4..069280d 100644 --- a/tools/biologix-public-intel/vercel/lib/collector.js +++ b/tools/biologix-public-intel/vercel/lib/collector.js @@ -99,6 +99,13 @@ function withQuery(url, values) { return parsed.toString(); } +export function freshPublicUrl(url, now = Date.now()) { + const bucketMs = POLL_INTERVAL_MINUTES * 60_000; + return withQuery(url, { + ovo_intel_poll: Math.floor(now / bucketMs), + }); +} + async function fetchJsonCollection(url) { const first = await fetchBounded(withQuery(url, { page: 1 }), { accept: "application/json", @@ -145,6 +152,7 @@ async function fetchJsonCollection(url) { 10, ), first_page_duration_ms: first.duration_ms, + headers: publicHeaders(first.response), }, }; } @@ -344,9 +352,18 @@ async function collectDeepSignals(homepage) { export async function collectPublicSnapshot(previousState, trigger) { const started = Date.now(); - const parentUrl = `${BIOLOGIX_BASE_URL}/wp-json/wc/store/v1/products?per_page=100&orderby=popularity&order=desc`; - const variationUrl = `${BIOLOGIX_BASE_URL}/wp-json/wc/store/v1/products?per_page=100&type=variation`; - const wpProductUrl = `${BIOLOGIX_BASE_URL}/wp-json/wp/v2/product?per_page=100&_fields=id,modified_gmt`; + const parentUrl = freshPublicUrl( + `${BIOLOGIX_BASE_URL}/wp-json/wc/store/v1/products?per_page=100&orderby=popularity&order=desc`, + started, + ); + const variationUrl = freshPublicUrl( + `${BIOLOGIX_BASE_URL}/wp-json/wc/store/v1/products?per_page=100&type=variation`, + started, + ); + const wpProductUrl = freshPublicUrl( + `${BIOLOGIX_BASE_URL}/wp-json/wp/v2/product?per_page=100&_fields=id,modified_gmt`, + started, + ); const [parents, variations, wpProducts, homepage] = await Promise.all([ fetchJsonCollection(parentUrl), fetchJsonCollection(variationUrl), @@ -393,6 +410,11 @@ export async function collectPublicSnapshot(previousState, trigger) { parent_count: parents.records.length, variation_count: variations.records.length, ...inventory, + public_catalog_sources: { + products: parents.meta, + variations: variations.meta, + modification_times: wpProducts.meta, + }, homepage: { status: homepage.status, duration_ms: homepage.duration_ms, diff --git a/tools/biologix-public-intel/vercel/test/biologix-intel.test.mjs b/tools/biologix-public-intel/vercel/test/biologix-intel.test.mjs index b4ec825..1617b7d 100644 --- a/tools/biologix-public-intel/vercel/test/biologix-intel.test.mjs +++ b/tools/biologix-public-intel/vercel/test/biologix-intel.test.mjs @@ -13,6 +13,7 @@ import { parseStockQuantity, summarizeUrlset, } from "../lib/biologix-intel-core.js"; +import { freshPublicUrl } from "../lib/collector.js"; function product(overrides = {}) { return { @@ -38,6 +39,18 @@ test("stock parsing distinguishes exact, unknown, and out of stock", () => { assert.equal(parseStockQuantity("Out of stock", false), 0); }); +test("public catalog URLs rotate once per poll window to bypass stale caches", () => { + const url = "https://example.com/wp-json/wc/store/v1/products?per_page=100"; + const first = freshPublicUrl(url, Date.UTC(2026, 6, 25, 12, 1)); + const sameWindow = freshPublicUrl(url, Date.UTC(2026, 6, 25, 12, 14)); + const nextWindow = freshPublicUrl(url, Date.UTC(2026, 6, 25, 12, 15)); + + assert.equal(first, sameWindow); + assert.notEqual(first, nextWindow); + assert.equal(new URL(first).searchParams.get("per_page"), "100"); + assert.ok(new URL(first).searchParams.has("ovo_intel_poll")); +}); + test("variable parent stock is not double-counted when children are exact", () => { const parent = product({ id: 10,