diff --git a/app/build.gradle.kts b/app/build.gradle.kts index d5d05b5..61c983c 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -1,3 +1,4 @@ +import groovy.json.JsonSlurper import java.util.Properties import org.jetbrains.kotlin.gradle.dsl.JvmTarget @@ -12,9 +13,9 @@ fun String.asBuildConfigStringLiteral(): String = val releaseVersionCode = providers.gradleProperty("ZEST_VERSION_CODE") .map(String::toInt) - .orElse(1) + .orElse(2) val releaseVersionName = providers.gradleProperty("ZEST_VERSION_NAME") - .orElse("1.0.0") + .orElse("1.0.1") val localProperties = Properties().apply { val localPropertiesFile = rootProject.layout.projectDirectory.file("local.properties").asFile if (localPropertiesFile.isFile) { @@ -240,10 +241,123 @@ val verifySessionOnlyStorage = tasks.register("verifySessionOnlyStorage") { } } +val verifyModuleVersionManifest = tasks.register("verifyModuleVersionManifest") { + group = "verification" + description = "Fails if a Zest module is missing from the production version manifest or docs." + doLast { + val manifestFile = rootProject.file("backend/module_versions.json") + val documentationFile = rootProject.file("documentation/12-module-versioning.md") + if (!manifestFile.isFile) { + throw GradleException("Missing module version manifest: ${manifestFile.relativeTo(rootDir)}") + } + if (!documentationFile.isFile) { + throw GradleException( + "Missing module version documentation: ${documentationFile.relativeTo(rootDir)}" + ) + } + + val manifest = JsonSlurper().parse(manifestFile) as Map<*, *> + val releaseVersion = manifest["releaseVersion"] as? String + ?: throw GradleException("module_versions.json must define releaseVersion.") + val versionPattern = Regex("""^\d+\.\d+\.\d+([+-][0-9A-Za-z.-]+)?$""") + if (!versionPattern.matches(releaseVersion)) { + throw GradleException("releaseVersion must be semantic versioning: $releaseVersion") + } + + val modules = manifest["modules"] as? List<*> + ?: throw GradleException("module_versions.json must define modules[].") + val moduleMaps = modules.mapIndexed { index, item -> + item as? Map<*, *> ?: throw GradleException("modules[$index] must be a JSON object.") + } + val moduleIds = mutableSetOf() + val docsText = documentationFile.readText() + + fun moduleValue(module: Map<*, *>, key: String): String = + module[key] as? String + ?: throw GradleException("Every module must define string field '$key': $module") + + moduleMaps.forEach { module -> + val id = moduleValue(module, "id") + val kind = moduleValue(module, "kind") + val path = moduleValue(module, "path") + val version = moduleValue(module, "version") + + if (!moduleIds.add(id)) { + throw GradleException("Duplicate module version entry: $id") + } + if (!versionPattern.matches(version)) { + throw GradleException("Module $id has non-semver version: $version") + } + if (!docsText.contains("`$id`") || !docsText.contains("`$version`")) { + throw GradleException( + "Module version documentation is missing module $id version $version." + ) + } + if (kind != "android-gradle-module" && path.startsWith(":")) { + throw GradleException("Only android-gradle-module entries can use Gradle paths: $id") + } + if (!path.startsWith(":") && !rootProject.file(path).exists()) { + throw GradleException("Module $id points to missing path: $path") + } + } + + val manifestGradlePaths = moduleMaps + .filter { moduleValue(it, "kind") == "android-gradle-module" } + .map { moduleValue(it, "path") } + .toSet() + val actualGradlePaths = rootProject.allprojects + .map { it.path } + .filter { it != ":" } + .toSet() + val missingGradleModules = actualGradlePaths - manifestGradlePaths + if (missingGradleModules.isNotEmpty()) { + throw GradleException( + "Gradle modules missing from backend/module_versions.json: " + + missingGradleModules.joinToString() + ) + } + + val packageRoot = layout.projectDirectory + .dir("src/main/java/com/b2/ultraprocessed") + .asFile + val actualAndroidPackagePaths = packageRoot + .listFiles { file -> file.isDirectory } + .orEmpty() + .map { it.relativeTo(rootDir).invariantSeparatorsPath } + .toSet() + val manifestAndroidPackagePaths = moduleMaps + .filter { moduleValue(it, "kind") == "android-package" } + .map { moduleValue(it, "path") } + .toSet() + val missingAndroidPackages = actualAndroidPackagePaths - manifestAndroidPackagePaths + val staleAndroidPackages = manifestAndroidPackagePaths - actualAndroidPackagePaths + if (missingAndroidPackages.isNotEmpty() || staleAndroidPackages.isNotEmpty()) { + throw GradleException( + buildString { + append("Android package module version manifest mismatch.") + if (missingAndroidPackages.isNotEmpty()) { + append("\nMissing package entries: ") + append(missingAndroidPackages.joinToString()) + } + if (staleAndroidPackages.isNotEmpty()) { + append("\nStale package entries: ") + append(staleAndroidPackages.joinToString()) + } + } + ) + } + } +} + val verifySourceTreeForBuild = tasks.register("verifySourceTreeForBuild") { group = "verification" description = "Guards the Android source tree from retired files and macOS dataless placeholders." - dependsOn(verifyNoRetiredSourceFiles, verifyNoDatalessSources, verifySessionOnlyStorage) + dependsOn( + verifyNoRetiredSourceFiles, + verifyNoDatalessSources, + verifySessionOnlyStorage, + verifyModuleVersionManifest, + ) } tasks.named("preBuild") { diff --git a/app/src/main/java/com/b2/ultraprocessed/analysis/FoodAnalysisPipeline.kt b/app/src/main/java/com/b2/ultraprocessed/analysis/FoodAnalysisPipeline.kt index 5a6b8a0..e5d696e 100644 --- a/app/src/main/java/com/b2/ultraprocessed/analysis/FoodAnalysisPipeline.kt +++ b/app/src/main/java/com/b2/ultraprocessed/analysis/FoodAnalysisPipeline.kt @@ -305,7 +305,7 @@ class FoodAnalysisPipeline( companion object { const val MIN_NORMALIZED_LENGTH: Int = 12 - const val DEFAULT_MODEL_ID: String = "gemini-2.5-flash" + const val DEFAULT_MODEL_ID: String = "gemini-3.5-flash" private const val PROXY_ANALYSIS_TIMEOUT_MILLIS = 110_000L private const val CLASSIFICATION_TIMEOUT_MILLIS = PROXY_ANALYSIS_TIMEOUT_MILLIS private const val INGREDIENT_LIST_TIMEOUT_MILLIS = PROXY_ANALYSIS_TIMEOUT_MILLIS diff --git a/app/src/main/java/com/b2/ultraprocessed/ui/AnalyzingScreen.kt b/app/src/main/java/com/b2/ultraprocessed/ui/AnalyzingScreen.kt index 06c7b59..a322da6 100644 --- a/app/src/main/java/com/b2/ultraprocessed/ui/AnalyzingScreen.kt +++ b/app/src/main/java/com/b2/ultraprocessed/ui/AnalyzingScreen.kt @@ -121,11 +121,13 @@ fun AnalyzingScreen( LaunchedEffect(scanSessionId) { statusMessage = null + currentStep = 0 + progress = 0f val startAt = System.currentTimeMillis() val outcome = runCatching { val onStage: (AnalysisStage) -> Unit = { stage -> currentStep = stage.stepIndex() - progress = stage.progressPercent() + progress = maxOf(progress, stage.progressPercent()) } val onStatus: (String) -> Unit = { message -> statusMessage = message @@ -173,6 +175,14 @@ fun AnalyzingScreen( ) } + LaunchedEffect(scanSessionId, currentStep) { + val ceiling = currentStep.progressCeiling() + while (progress < ceiling) { + delay(450L) + progress = minOf(ceiling, progress + currentStep.progressIncrement()) + } + } + val transition = rememberInfiniteTransition(label = "analysis-rings") val rotation by transition.animateFloat( initialValue = 0f, @@ -351,6 +361,22 @@ private fun AnalysisStage.progressPercent(): Float = when (this) { AnalysisStage.AnalysingImage -> 8f AnalysisStage.ExtractingIngredients -> 35f - AnalysisStage.AnalysingIngredients -> 72f + AnalysisStage.AnalysingIngredients -> 40f AnalysisStage.Completed -> 100f } + +private fun Int.progressCeiling(): Float = + when (this) { + 0 -> 34f + 1 -> 39f + 2 -> 96f + else -> 100f + } + +private fun Int.progressIncrement(): Float = + when (this) { + 0 -> 3.5f + 1 -> 1f + 2 -> 0.45f + else -> 0f + } diff --git a/app/src/main/java/com/b2/ultraprocessed/ui/AppModels.kt b/app/src/main/java/com/b2/ultraprocessed/ui/AppModels.kt index 6777147..2032476 100644 --- a/app/src/main/java/com/b2/ultraprocessed/ui/AppModels.kt +++ b/app/src/main/java/com/b2/ultraprocessed/ui/AppModels.kt @@ -94,8 +94,8 @@ enum class AppDestination { object AppCatalog { val modelOptions = listOf( ModelOption( - id = "gemini-2.5-flash", - name = "Gemini 2.5 Flash", + id = "gemini-3.5-flash", + name = "Gemini 3.5 Flash", provider = "Gemini (Google)", description = "Backend proxy analysis model for on-device OCR output", supportsImages = false, diff --git a/app/src/main/java/com/b2/ultraprocessed/ui/UltraProcessedApp.kt b/app/src/main/java/com/b2/ultraprocessed/ui/UltraProcessedApp.kt index 4eed5e3..fb2da9b 100644 --- a/app/src/main/java/com/b2/ultraprocessed/ui/UltraProcessedApp.kt +++ b/app/src/main/java/com/b2/ultraprocessed/ui/UltraProcessedApp.kt @@ -94,6 +94,13 @@ fun UltraProcessedApp( destination = nextDestination } + fun clearCurrentResultAndImage() { + deleteLocalScanImage(appContext, currentScanResult?.labelImagePath) + currentScanResult = null + lastCapturedPhotoPath = null + barcodeValue = null + } + fun navigateBackWithinApp() { when (destination) { AppDestination.Splash -> Unit @@ -107,8 +114,16 @@ fun UltraProcessedApp( } } AppDestination.Scanner -> Unit - AppDestination.Analyzing -> navigateTo(AppDestination.Scanner) - AppDestination.Results -> navigateTo(AppDestination.Scanner) + AppDestination.Analyzing -> { + deleteLocalScanImage(appContext, lastCapturedPhotoPath) + lastCapturedPhotoPath = null + barcodeValue = null + navigateTo(AppDestination.Scanner) + } + AppDestination.Results -> { + clearCurrentResultAndImage() + navigateTo(AppDestination.Scanner) + } AppDestination.AnalysisError -> { analysisErrorMessage = "" navigateTo(AppDestination.Scanner) @@ -223,8 +238,7 @@ fun UltraProcessedApp( ?: selectedModelId, onSuccess = { result -> barcodeValue = null - deleteLocalScanImage(appContext, result.labelImagePath) - currentScanResult = result.copy(labelImagePath = null) + currentScanResult = result lastCapturedPhotoPath = null playSound(AppSoundEvent.Success) navigateTo(AppDestination.Results) @@ -245,6 +259,7 @@ fun UltraProcessedApp( ResultsScreen( result = result, onScanAgain = { + clearCurrentResultAndImage() navigateTo(AppDestination.Scanner) }, chatEnabled = true, diff --git a/app/src/test/java/com/b2/ultraprocessed/network/llm/ProxyFoodLabelLlmWorkflowTest.kt b/app/src/test/java/com/b2/ultraprocessed/network/llm/ProxyFoodLabelLlmWorkflowTest.kt index 4b02785..0ad362f 100644 --- a/app/src/test/java/com/b2/ultraprocessed/network/llm/ProxyFoodLabelLlmWorkflowTest.kt +++ b/app/src/test/java/com/b2/ultraprocessed/network/llm/ProxyFoodLabelLlmWorkflowTest.kt @@ -117,7 +117,7 @@ class ProxyFoodLabelLlmWorkflowTest { .build() companion object { - private const val MODEL = "gemini-2.5-flash" + private const val MODEL = "gemini-3.5-flash" private val SUCCESS_BODY = """ { @@ -142,7 +142,7 @@ class ProxyFoodLabelLlmWorkflowTest { "confidence": 0.6, "warnings": [] }, - "model": "gemini-2.5-flash", + "model": "gemini-3.5-flash", "usage": {"inputTokens": 12, "outputTokens": 34, "totalTokens": 46} } """.trimIndent() diff --git a/app/src/test/java/com/b2/ultraprocessed/network/llm/ProxyResultChatWorkflowTest.kt b/app/src/test/java/com/b2/ultraprocessed/network/llm/ProxyResultChatWorkflowTest.kt index 542210f..6ebd776 100644 --- a/app/src/test/java/com/b2/ultraprocessed/network/llm/ProxyResultChatWorkflowTest.kt +++ b/app/src/test/java/com/b2/ultraprocessed/network/llm/ProxyResultChatWorkflowTest.kt @@ -41,7 +41,7 @@ class ProxyResultChatWorkflowTest { val reply = workflow.askAboutResult( result = chatContext, question = "What is concerning?", - modelId = "gemini-2.5-flash", + modelId = "gemini-3.5-flash", history = listOf( ResultChatHistoryMessage(role = "user", text = "Why NOVA 4?"), ResultChatHistoryMessage(role = "assistant", text = "Because additives were detected."), @@ -84,7 +84,7 @@ class ProxyResultChatWorkflowTest { workflow.askAboutResult( result = chatContext, question = "What is concerning?", - modelId = "gemini-2.5-flash", + modelId = "gemini-3.5-flash", ).getOrThrow() assertFalse(hasAuthorization) @@ -113,7 +113,7 @@ class ProxyResultChatWorkflowTest { val result = workflow.askAboutResult( result = chatContext, question = "What is concerning?", - modelId = "gemini-2.5-flash", + modelId = "gemini-3.5-flash", ) assertTrue(result.isFailure) @@ -145,7 +145,7 @@ class ProxyResultChatWorkflowTest { val result = workflow.askAboutResult( result = chatContext, question = "What is concerning?", - modelId = "gemini-2.5-flash", + modelId = "gemini-3.5-flash", ) val message = result.exceptionOrNull()?.message.orEmpty() @@ -187,7 +187,7 @@ class ProxyResultChatWorkflowTest { "answer": "Soy lecithin is the main marker.", "reason": "" }, - "model": "gemini-2.5-flash", + "model": "gemini-3.5-flash", "usage": {"inputTokens": 1, "outputTokens": 2, "totalTokens": 3} } """.trimIndent() diff --git a/backend/Dockerfile b/backend/Dockerfile index 4557f54..d30e1f1 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -9,7 +9,7 @@ WORKDIR /app COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt -COPY main.py prompt.py ./ +COPY main.py prompt.py module_versions.json ./ COPY prompts ./prompts EXPOSE 8080 diff --git a/backend/README.md b/backend/README.md index 9f4f001..b1551c3 100644 --- a/backend/README.md +++ b/backend/README.md @@ -58,7 +58,7 @@ integration is a direct deserialize: "confidence": 0.6, "warnings": [] }, - "model": "gemini-2.5-flash", + "model": "gemini-3.5-flash", "usage": { "inputTokens": 0, "outputTokens": 0, "totalTokens": 0 } } ``` @@ -98,7 +98,7 @@ Chat response: "answer": "Short scan-scoped answer.", "reason": "" }, - "model": "gemini-2.5-flash", + "model": "gemini-3.5-flash", "usage": { "inputTokens": 0, "outputTokens": 0, "totalTokens": 0 } } ``` @@ -111,9 +111,9 @@ On model/auth/timeout failure the service returns HTTP 502 with |-------------------|--------------------|------------------------------------------| | `GCP_PROJECT_ID` | `b2-ultra-processed` | GCP project for Vertex AI | | `GCP_LOCATION` | `us-east1` | Vertex AI region | -| `GEMINI_MODEL` | `gemini-2.5-flash` | Model id | +| `GEMINI_MODEL` | `gemini-3.5-flash` | Model id | | `GEMINI_TIMEOUT_MS` | `30000` | Per-request model timeout (ms) | -| `GEMINI_MAX_OUTPUT_TOKENS` | `4096` | Analysis output cap (shared by gemini-2.5-flash thinking + JSON answer) | +| `GEMINI_MAX_OUTPUT_TOKENS` | `4096` | Analysis output cap (shared by thinking + JSON answer) | | `GEMINI_CHAT_MAX_OUTPUT_TOKENS` | `2048` | Result-chat output cap (shared by thinking + answer) | | `ENABLE_OPENAPI_DOCS` | `false` | Enables `/docs`, `/redoc`, and `/openapi.json` only for local/dev use | | `CORS_ALLOWED_ORIGINS` | empty | Comma-separated browser origins allowed to call the API; empty disables CORS middleware | @@ -154,18 +154,25 @@ curl -X POST http://localhost:8080/chat \ -d '{"question":"Which ingredient is the biggest concern?","result":{"productName":"Choco Wafer","novaGroup":4,"summary":"Contains additive markers.","sourceLabel":"OCR","confidence":0.85,"ingredients":["sugar","soy lecithin"],"ingredientAssessments":[{"name":"soy lecithin","verdict":"nova-4","reason":"Emulsifier marker."}],"allergens":["soy"],"warnings":[]},"history":[]}' ``` -### Latency benchmark +### Latency and NOVA benchmark Unit tests prove contract and single-call behavior, not production p95. Use the benchmark against a -deployed service before claiming the LLM-only path meets the p95 target: +deployed service before claiming the LLM-only path meets the p95 target or expected NOVA behavior: ```bash python backend/benchmark_analyze.py \ https://ultraprocessed-ai-proxy-894254677159.us-east1.run.app \ - --iterations 5 + --iterations 1 \ + --details ``` -The script prints success rate, p50, p95, p99, and HTTP error counts for a fixed OCR corpus. +The benchmark corpus contains 40 Open Food Facts products: 10 expected NOVA 1, 10 expected NOVA 2, +10 expected NOVA 3, and 10 expected NOVA 4. Expected labels come from Open Food Facts `nova_group` +values, and each corpus row includes the source product code and URL. The script prints success rate, NOVA accuracy, +per-group accuracy, p50, p95, p99, token usage when returned by the backend, and HTTP/model error +counts. Add `--strict-nova` to exit non-zero when any NOVA mismatch is observed. + +Corpus source: Open Food Facts public database and API. ## Deploy to Cloud Run @@ -176,7 +183,7 @@ gcloud run deploy ultraprocessed-ai-proxy \ --region us-east1 \ --service-account up-app-service@b2-ultra-processed.iam.gserviceaccount.com \ --allow-unauthenticated \ - --set-env-vars GCP_PROJECT_ID=b2-ultra-processed,GCP_LOCATION=us-east1,GEMINI_MODEL=gemini-2.5-flash + --set-env-vars GCP_PROJECT_ID=b2-ultra-processed,GCP_LOCATION=us-east1,GEMINI_MODEL=gemini-3.5-flash ``` The service account must have Vertex AI access (e.g. `roles/aiplatform.user`) on the project. @@ -199,7 +206,7 @@ The service account must have Vertex AI access (e.g. `roles/aiplatform.user`) on ## Notes / caveats -- **Region + model:** Vertex AI regional availability for `gemini-2.5-flash` varies. If a deploy +- **Region + model:** Vertex AI regional availability for `gemini-3.5-flash` varies. If a deploy reports the model is unavailable in `us-east1`, set `GCP_LOCATION=global` (or `us-central1`) via `--set-env-vars` and redeploy. - **Backend-owned prompts:** Android sends scan data and chat questions only. The analysis and chat prompts live diff --git a/backend/benchmark_analyze.py b/backend/benchmark_analyze.py index 3f18f07..a784469 100644 --- a/backend/benchmark_analyze.py +++ b/backend/benchmark_analyze.py @@ -1,7 +1,7 @@ -"""Measure deployed /analyze latency against a small fixed OCR corpus. +"""Measure deployed /analyze latency and NOVA accuracy against a fixed corpus. Usage: - python benchmark_analyze.py https://example.run.app --iterations 3 + python backend/benchmark_analyze.py https://example.run.app --iterations 1 """ from __future__ import annotations @@ -13,39 +13,11 @@ import time import urllib.error import urllib.request +from pathlib import Path +from typing import Any, Callable -CORPUS = [ - { - "product_name": "Simple oats", - "ingredient_text": "whole grain oats", - }, - { - "product_name": "Chocolate cereal", - "ingredient_text": ( - "whole grain oats, sugar, corn syrup, salt, natural flavor, soy lecithin, " - "caramel color, vitamins and minerals" - ), - }, - { - "product_name": "Noisy OCR snack", - "ingredient_text": ( - "INGREDIENTS: wheat flour; sugar; vegetable oil (canola, palm); cocoa powder; " - "soy lecithin. NUTRITION FACTS serving size calories barcode 123456" - ), - }, - { - "product_name": "Allergen dense bar", - "ingredient_text": ( - "almond flour, wheat flour, milk powder, egg white, sesame seed, soy protein isolate, " - "peanut butter, natural flavor" - ), - }, - { - "product_name": "Non-food text", - "ingredient_text": "white painted wall, window frame, barcode 00881234, recycle symbol", - }, -] +DEFAULT_CORPUS_PATH = Path(__file__).with_name("nova_benchmark_corpus.json") def percentile(values: list[float], pct: float) -> float: @@ -56,7 +28,24 @@ def percentile(values: list[float], pct: float) -> float: return ordered[index] -def post_json(url: str, payload: dict, timeout: float) -> tuple[int, str]: +def load_corpus(path: Path = DEFAULT_CORPUS_PATH) -> list[dict[str, Any]]: + with path.open(encoding="utf-8") as corpus_file: + corpus = json.load(corpus_file) + if not isinstance(corpus, list): + raise ValueError("Benchmark corpus must be a JSON array.") + for index, sample in enumerate(corpus): + if not isinstance(sample, dict): + raise ValueError(f"Corpus item {index} must be an object.") + for field in ("id", "product_name", "ingredient_text", "expected_nova_group"): + if field not in sample: + raise ValueError(f"Corpus item {index} missing field: {field}") + expected = sample["expected_nova_group"] + if expected not in (1, 2, 3, 4): + raise ValueError(f"Corpus item {sample['id']} has invalid expected_nova_group: {expected}") + return corpus + + +def post_json(url: str, payload: dict[str, Any], timeout: float) -> tuple[int, str]: data = json.dumps(payload).encode("utf-8") request = urllib.request.Request( url, @@ -71,47 +60,151 @@ def post_json(url: str, payload: dict, timeout: float) -> tuple[int, str]: return exc.code, exc.read().decode("utf-8", errors="replace") -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("base_url", help="Cloud Run base URL or full /analyze URL") - parser.add_argument("--iterations", type=int, default=3) - parser.add_argument("--timeout", type=float, default=30.0) - args = parser.parse_args() - - analyze_url = args.base_url.rstrip("/") +def normalize_analyze_url(base_url: str) -> str: + analyze_url = base_url.rstrip("/") if not analyze_url.endswith("/analyze"): analyze_url += "/analyze" + return analyze_url + + +def nova_group_from_body(body: str) -> int | None: + try: + payload = json.loads(body) + except json.JSONDecodeError: + return None + nova = payload.get("nova") + if not isinstance(nova, dict): + return None + group = nova.get("novaGroup") + return group if isinstance(group, int) else None + + +def usage_from_body(body: str) -> dict[str, int]: + try: + payload = json.loads(body) + except json.JSONDecodeError: + return {} + usage = payload.get("usage") + if not isinstance(usage, dict): + return {} + result: dict[str, int] = {} + for key in ("inputTokens", "outputTokens", "totalTokens"): + value = usage.get(key) + if isinstance(value, int): + result[key] = value + if {"inputTokens", "outputTokens", "totalTokens"} <= result.keys(): + result["estimatedHiddenThinkingTokens"] = ( + result["totalTokens"] - result["inputTokens"] - result["outputTokens"] + ) + return result + + +def summarize_group(rows: list[dict[str, Any]], expected_group: int) -> dict[str, Any]: + group_rows = [row for row in rows if row["expectedNovaGroup"] == expected_group] + latencies = [row["latencyMs"] for row in group_rows if row["httpStatus"] == 200] + correct = sum(1 for row in group_rows if row.get("correct") is True) + return { + "samples": len(group_rows), + "correct": correct, + "accuracy": round(correct / len(group_rows), 4) if group_rows else 0, + "p50Ms": round(statistics.median(latencies), 2) if latencies else 0, + "p95Ms": round(percentile(latencies, 95), 2) if latencies else 0, + } - latencies: list[float] = [] + +def run_benchmark( + analyze_url: str, + corpus: list[dict[str, Any]], + iterations: int, + timeout: float, + post_fn: Callable[[str, dict[str, Any], float], tuple[int, str]] = post_json, +) -> dict[str, Any]: + rows: list[dict[str, Any]] = [] errors: dict[str, int] = {} - total = 0 - for _ in range(args.iterations): - for sample in CORPUS: - total += 1 - payload = {"type": "analysis", **sample} + + for iteration in range(1, iterations + 1): + for sample in corpus: + expected = int(sample["expected_nova_group"]) + payload = { + "type": "analysis", + "product_name": sample["product_name"], + "ingredient_text": sample["ingredient_text"], + } start = time.perf_counter() - status, body = post_json(analyze_url, payload, args.timeout) + status, body = post_fn(analyze_url, payload, timeout) elapsed_ms = (time.perf_counter() - start) * 1000 - if 200 <= status < 300: - latencies.append(elapsed_ms) - else: + actual = nova_group_from_body(body) if 200 <= status < 300 else None + correct = actual == expected + + if status < 200 or status >= 300: key = f"HTTP {status}" errors[key] = errors.get(key, 0) + 1 - print(f"{key}: {body[:300]}", file=sys.stderr) - - success = len(latencies) - result = { + print(f"{key} {sample['id']}: {body[:300]}", file=sys.stderr) + elif actual is None: + errors["missing_nova_group"] = errors.get("missing_nova_group", 0) + 1 + elif not correct: + errors["nova_mismatch"] = errors.get("nova_mismatch", 0) + 1 + + row = { + "iteration": iteration, + "id": sample["id"], + "productName": sample["product_name"], + "expectedNovaGroup": expected, + "actualNovaGroup": actual, + "correct": correct, + "httpStatus": status, + "latencyMs": round(elapsed_ms, 2), + } + row.update(usage_from_body(body)) + rows.append(row) + + success_rows = [row for row in rows if row["httpStatus"] == 200] + latencies = [row["latencyMs"] for row in success_rows] + correct = sum(1 for row in success_rows if row.get("correct") is True) + + return { "url": analyze_url, - "totalRequests": total, - "successfulRequests": success, - "successRate": round(success / total, 4) if total else 0, + "corpusSize": len(corpus), + "iterations": iterations, + "totalRequests": len(rows), + "successfulRequests": len(success_rows), + "successRate": round(len(success_rows) / len(rows), 4) if rows else 0, + "novaCorrect": correct, + "novaAccuracy": round(correct / len(success_rows), 4) if success_rows else 0, "p50Ms": round(statistics.median(latencies), 2) if latencies else 0, "p95Ms": round(percentile(latencies, 95), 2), "p99Ms": round(percentile(latencies, 99), 2), + "byExpectedNovaGroup": { + str(group): summarize_group(rows, group) for group in (1, 2, 3, 4) + }, "errors": errors, + "results": rows, } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("base_url", help="Cloud Run base URL or full /analyze URL") + parser.add_argument("--iterations", type=int, default=1) + parser.add_argument("--timeout", type=float, default=60.0) + parser.add_argument("--corpus", type=Path, default=DEFAULT_CORPUS_PATH) + parser.add_argument("--details", action="store_true", help="Include per-product rows.") + parser.add_argument("--strict-nova", action="store_true", help="Exit non-zero on NOVA mismatch.") + args = parser.parse_args() + + corpus = load_corpus(args.corpus) + result = run_benchmark( + analyze_url=normalize_analyze_url(args.base_url), + corpus=corpus, + iterations=args.iterations, + timeout=args.timeout, + ) + if not args.details: + result = {key: value for key, value in result.items() if key != "results"} print(json.dumps(result, indent=2)) - return 0 if not errors else 1 + has_http_errors = any(key.startswith("HTTP ") for key in result["errors"]) + has_nova_errors = "nova_mismatch" in result["errors"] or "missing_nova_group" in result["errors"] + return 1 if has_http_errors or (args.strict_nova and has_nova_errors) else 0 if __name__ == "__main__": diff --git a/backend/main.py b/backend/main.py index c0a16e0..78385a3 100644 --- a/backend/main.py +++ b/backend/main.py @@ -14,6 +14,7 @@ import logging import os import re +from pathlib import Path from typing import Any, Optional from fastapi import FastAPI, HTTPException @@ -31,9 +32,9 @@ # --- Configuration (env vars with spec defaults) --------------------------------- GCP_PROJECT_ID = os.getenv("GCP_PROJECT_ID", "b2-ultra-processed") GCP_LOCATION = os.getenv("GCP_LOCATION", "us-east1") -GEMINI_MODEL = os.getenv("GEMINI_MODEL", "gemini-2.5-flash") +GEMINI_MODEL = os.getenv("GEMINI_MODEL", "gemini-3.5-flash") REQUEST_TIMEOUT_MS = int(os.getenv("GEMINI_TIMEOUT_MS", "30000")) -# ponytail: gemini-2.5-flash "thinking" tokens share this output budget. 700/512 let thinking +# Gemini thinking tokens share this output budget. Low caps can let thinking # eat the whole cap on complex labels, truncating the JSON to garbage -> 502 model_response_unparseable. # Raised to comfortably fit thinking + answer for real OCR inputs. If a pathological label still # truncates, bound thinking with types.ThinkingConfig(thinking_budget=N) on the generate_content calls. @@ -51,10 +52,23 @@ for origin in os.getenv("CORS_ALLOWED_ORIGINS", "").split(",") if origin.strip() ] +MODULE_VERSION_MANIFEST_PATH = Path(__file__).with_name("module_versions.json") + + +def load_module_version_manifest() -> dict[str, Any]: + with MODULE_VERSION_MANIFEST_PATH.open(encoding="utf-8") as manifest_file: + manifest = json.load(manifest_file) + if not isinstance(manifest, dict): + raise RuntimeError("module version manifest must be a JSON object") + return manifest + + +MODULE_VERSION_MANIFEST = load_module_version_manifest() +SERVICE_VERSION = str(MODULE_VERSION_MANIFEST.get("releaseVersion", "0.0.0")) app = FastAPI( title="Ultraprocessed AI Proxy", - version="1.0.0", + version=SERVICE_VERSION, docs_url="/docs" if ENABLE_DOCS else None, redoc_url="/redoc" if ENABLE_DOCS else None, openapi_url="/openapi.json" if ENABLE_DOCS else None, @@ -505,6 +519,11 @@ def healthz() -> dict: return {"status": "ok"} +@app.get("/version") +def version() -> dict: + return MODULE_VERSION_MANIFEST + + @app.post("/analyze") def analyze(req: AnalyzeRequest) -> dict: try: diff --git a/backend/module_versions.json b/backend/module_versions.json new file mode 100644 index 0000000..01b33a8 --- /dev/null +++ b/backend/module_versions.json @@ -0,0 +1,91 @@ +{ + "schemaVersion": 1, + "product": "Zest", + "releaseVersion": "1.0.1", + "modules": [ + { + "id": "android-app", + "kind": "android-gradle-module", + "path": ":app", + "version": "1.0.1", + "runtimeSurface": "Android BuildConfig.VERSION_NAME" + }, + { + "id": "backend-proxy", + "kind": "cloud-run-service", + "path": "backend", + "version": "1.0.1", + "runtimeSurface": "FastAPI app.version and GET /version" + }, + { + "id": "analysis", + "kind": "android-package", + "path": "app/src/main/java/com/b2/ultraprocessed/analysis", + "version": "1.0.1", + "runtimeSurface": "module version manifest" + }, + { + "id": "barcode", + "kind": "android-package", + "path": "app/src/main/java/com/b2/ultraprocessed/barcode", + "version": "1.0.0", + "runtimeSurface": "module version manifest" + }, + { + "id": "camera", + "kind": "android-package", + "path": "app/src/main/java/com/b2/ultraprocessed/camera", + "version": "1.0.0", + "runtimeSurface": "module version manifest" + }, + { + "id": "classify", + "kind": "android-package", + "path": "app/src/main/java/com/b2/ultraprocessed/classify", + "version": "1.0.0", + "runtimeSurface": "module version manifest" + }, + { + "id": "ingredients", + "kind": "android-package", + "path": "app/src/main/java/com/b2/ultraprocessed/ingredients", + "version": "1.0.0", + "runtimeSurface": "module version manifest" + }, + { + "id": "network", + "kind": "android-package", + "path": "app/src/main/java/com/b2/ultraprocessed/network", + "version": "1.0.0", + "runtimeSurface": "module version manifest" + }, + { + "id": "ocr", + "kind": "android-package", + "path": "app/src/main/java/com/b2/ultraprocessed/ocr", + "version": "1.0.0", + "runtimeSurface": "module version manifest" + }, + { + "id": "storage", + "kind": "android-package", + "path": "app/src/main/java/com/b2/ultraprocessed/storage", + "version": "1.0.0", + "runtimeSurface": "module version manifest" + }, + { + "id": "ui", + "kind": "android-package", + "path": "app/src/main/java/com/b2/ultraprocessed/ui", + "version": "1.0.1", + "runtimeSurface": "module version manifest" + }, + { + "id": "backend-prompts", + "kind": "backend-contract-assets", + "path": "backend/prompts", + "version": "1.0.3", + "runtimeSurface": "backend-owned prompt files" + } + ] +} diff --git a/backend/nova_benchmark_corpus.json b/backend/nova_benchmark_corpus.json new file mode 100644 index 0000000..01e6386 --- /dev/null +++ b/backend/nova_benchmark_corpus.json @@ -0,0 +1,402 @@ +[ + { + "id": "off-nova1-01-6111035002175", + "expected_nova_group": 1, + "product_name": "Sidi Ali", + "ingredient_text": "Sodium, Calcium, Magnésium, Potassium, Bicarbonates, Sulfates, Chlorures,", + "source": "Open Food Facts", + "source_code": "6111035002175", + "source_url": "https://world.openfoodfacts.org/product/6111035002175/sidi-ali", + "brands": "Sidi Ali" + }, + { + "id": "off-nova1-02-5010477348630", + "expected_nova_group": 1, + "product_name": "Muesli 36% fruits noix & graines", + "ingredient_text": "Céréales complètes (64%) (flocons de blé, flocons d'avoine, pétales de blé grillé (13%)), raisins de sultane (16%), raisins secs (11%), copeaux de noix de coco (3,5%), noisettes grillées entières (3%), graines de tournesol (2,5%)", + "source": "Open Food Facts", + "source_code": "5010477348630", + "source_url": "https://world.openfoodfacts.org/product/5010477348630/muesli-36-fruits-noix-graines-jordan", + "brands": "Jordan" + }, + { + "id": "off-nova1-03-20004132", + "expected_nova_group": 1, + "product_name": "Gehackte Tomaten in Tomatensaft", + "ingredient_text": "60% chopped tomatoes, 39,9% tomato juice, acidity regulator: citric acid", + "source": "Open Food Facts", + "source_code": "20004132", + "source_url": "https://world.openfoodfacts.org/product/20004132/gehackte-tomaten-in-tomatensaft-baresa", + "brands": "Baresa" + }, + { + "id": "off-nova1-04-6111242101180", + "expected_nova_group": 1, + "product_name": "uht jaouda 1L", + "ingredient_text": "Lait entier", + "source": "Open Food Facts", + "source_code": "6111242101180", + "source_url": "https://world.openfoodfacts.org/product/6111242101180/uht-jaouda-1l", + "brands": "Jaouda" + }, + { + "id": "off-nova1-05-5449000061515", + "expected_nova_group": 1, + "product_name": "Ciel", + "ingredient_text": "eau minérale, calcium, magnésium, sodium, potassium, bicarbonates, chlorures, sulfates, nitrates", + "source": "Open Food Facts", + "source_code": "5449000061515", + "source_url": "https://world.openfoodfacts.org/product/5449000061515/ciel-the-coca-cola-company", + "brands": "THE COCA-COLA COMPANY" + }, + { + "id": "off-nova1-06-6111035001383", + "expected_nova_group": 1, + "product_name": "Eau minerale", + "ingredient_text": "Magnesium, Potassium, Bicarbonates, Sodium, Sulfates, Chlorides, Nitrates", + "source": "Open Food Facts", + "source_code": "6111035001383", + "source_url": "https://world.openfoodfacts.org/product/6111035001383/eau-minerale-sidi-ali", + "brands": "Sidi ali" + }, + { + "id": "off-nova1-07-3760020507350", + "expected_nova_group": 1, + "product_name": "Pur beurre de cacahuète", + "ingredient_text": "CACAHUETES issues de l'agriculture biologique. Peut contenir des traces de FRUITS A COQUE et de graines de SESAME.", + "source": "Open Food Facts", + "source_code": "3760020507350", + "source_url": "https://world.openfoodfacts.org/product/3760020507350/pure-peanut-butter-jardin-bio-etic", + "brands": "Jardin Bio étic" + }, + { + "id": "off-nova1-08-5010477348678", + "expected_nova_group": 1, + "product_name": "Special Muesli 30% fruits & noix", + "ingredient_text": "Céréales complètes (59%) (flocons d'_avoine_, flocons de _blé_, flocons de _blé_ grillés), fruits secs (30%) (raisins secs, raisins de sultanine, abricots secs hachés, tranches de pommes séchées, éclats de noix de coco grillés, noix_noisettes_ hachées grillées, _amandes_ effilées), flocons d'_orge_, graines de tournesol.", + "source": "Open Food Facts", + "source_code": "5010477348678", + "source_url": "https://world.openfoodfacts.org/product/5010477348678/special-muesli-jordans", + "brands": "Jordans" + }, + { + "id": "off-nova1-09-20005733", + "expected_nova_group": 1, + "product_name": "Cerneaux de Noix", + "ingredient_text": "noix décortiquée Traces éventuelles d'autres fruits à coque et d'arachides.", + "source": "Open Food Facts", + "source_code": "20005733", + "source_url": "https://world.openfoodfacts.org/product/20005733/walnusse-alesto", + "brands": "Alesto" + }, + { + "id": "off-nova1-10-20003166", + "expected_nova_group": 1, + "product_name": "FIOCCHI D'AVENA", + "ingredient_text": "100 % wholemeal oat flakes", + "source": "Open Food Facts", + "source_code": "20003166", + "source_url": "https://world.openfoodfacts.org/product/20003166/fiocchi-d-avena-crownfield", + "brands": "Crownfield" + }, + { + "id": "off-nova2-01-3451790988677", + "expected_nova_group": 2, + "product_name": "Le Beurre Tendre", + "ingredient_text": "cream (origin france), milk fat , lactic ferments,", + "source": "Open Food Facts", + "source_code": "3451790988677", + "source_url": "https://world.openfoodfacts.org/product/3451790988677/le-beurre-tendre-elle-vire", + "brands": "Elle&Vire" + }, + { + "id": "off-nova2-02-6191509900671", + "expected_nova_group": 2, + "product_name": "EXTRA VIRGIN OLIVE OIL", + "ingredient_text": "EXTRA VIRGIN OLIVE OIL", + "source": "Open Food Facts", + "source_code": "6191509900671", + "source_url": "https://world.openfoodfacts.org/product/6191509900671/extra-virgin-olive-oil-terra-delyssa", + "brands": "TERRA DELYSSA" + }, + { + "id": "off-nova2-03-6111184001562", + "expected_nova_group": 2, + "product_name": "Sel de table iodé", + "ingredient_text": "Sel, iodate de potassium et anti-agglomérants", + "source": "Open Food Facts", + "source_code": "6111184001562", + "source_url": "https://world.openfoodfacts.org/product/6111184001562/sel-de-table-iode-star", + "brands": "Star" + }, + { + "id": "off-nova2-04-4056489166856", + "expected_nova_group": 2, + "product_name": "Olivenöl", + "ingredient_text": "Natives Olivenöl extra", + "source": "Open Food Facts", + "source_code": "4056489166856", + "source_url": "https://world.openfoodfacts.org/product/4056489166856/natives-olivenol-primadonna", + "brands": "Primadonna" + }, + { + "id": "off-nova2-05-6111180000392", + "expected_nova_group": 2, + "product_name": "سكر الفانيلين", + "ingredient_text": "sucre, amidon de maïs, éthylvanilline", + "source": "Open Food Facts", + "source_code": "6111180000392", + "source_url": "https://world.openfoodfacts.org/product/6111180000392/ideal-vanillin-sugar", + "brands": "Idéal" + }, + { + "id": "off-nova2-06-5740900404465", + "expected_nova_group": 2, + "product_name": "Spreadable", + "ingredient_text": "butter (64%) (milk), rapeseed oil, water, lactic culture (milk), salt", + "source": "Open Food Facts", + "source_code": "5740900404465", + "source_url": "https://world.openfoodfacts.org/product/5740900404465/spreadable-lurpak", + "brands": "Lurpak" + }, + { + "id": "off-nova2-07-6111024004746", + "expected_nova_group": 2, + "product_name": "Huile d'olive vierge de Maroc", + "ingredient_text": "Huile d'olive.", + "source": "Open Food Facts", + "source_code": "6111024004746", + "source_url": "https://world.openfoodfacts.org/product/6111024004746/huile-d-olive-vierge-de-maroc-alhorra", + "brands": "ALHORRA" + }, + { + "id": "off-nova2-08-3165433724019", + "expected_nova_group": 2, + "product_name": "Cassonade pure canne", + "ingredient_text": "100% pure brown sugar", + "source": "Open Food Facts", + "source_code": "3165433724019", + "source_url": "https://world.openfoodfacts.org/product/3165433724019/cassonade-pure-canne-daddy", + "brands": "Daddy" + }, + { + "id": "off-nova2-09-6191509900855", + "expected_nova_group": 2, + "product_name": "Organic Extra Virgin Olive Oil", + "ingredient_text": "extra virgin olive oil (organic of higher grade obtained directly from olives and only by mechanical processes),", + "source": "Open Food Facts", + "source_code": "6191509900855", + "source_url": "https://world.openfoodfacts.org/product/6191509900855/organic-extra-virgin-olive-oil-terra-delyssa", + "brands": "Terra Delyssa" + }, + { + "id": "off-nova2-10-4056489424925", + "expected_nova_group": 2, + "product_name": "Smooth Peanut Butter", + "ingredient_text": "94% cacahuète, huile de palme, 1,2% huile d'arachides, sucre de canne, sel. Traces éventuelles de soja", + "source": "Open Food Facts", + "source_code": "4056489424925", + "source_url": "https://world.openfoodfacts.org/product/4056489424925/peanut-butter-creamy-maribel", + "brands": "Maribel" + }, + { + "id": "off-nova3-01-8715700407760", + "expected_nova_group": 3, + "product_name": "Tomato Ketchup BIO", + "ingredient_text": "Tomates (180 g pour 100 g de Ketchup), sucre, vinaigre, sel, épices (contient céleri), poudre d'oignon, poudre d'ail,", + "source": "Open Food Facts", + "source_code": "8715700407760", + "source_url": "https://world.openfoodfacts.org/product/8715700407760/tomato-ketchup-bio-heinz", + "brands": "Heinz" + }, + { + "id": "off-nova3-02-5025125000006", + "expected_nova_group": 3, + "product_name": "White Ciabattin", + "ingredient_text": "Wheat Flour (Wheat Flour, Calcium Carbonate, Folic Acid Iron, Niacin, Thiamin),Water, Salt, Fermented Wheat Flour.", + "source": "Open Food Facts", + "source_code": "5025125000006", + "source_url": "https://world.openfoodfacts.org/product/5025125000006/white-ciabattin-jason-s-sourdough", + "brands": "Jason's Sourdough" + }, + { + "id": "off-nova3-03-6111242100992", + "expected_nova_group": 3, + "product_name": "Perly", + "ingredient_text": "milk cream, cream, sugar, banana, bacteria", + "source": "Open Food Facts", + "source_code": "6111242100992", + "source_url": "https://world.openfoodfacts.org/product/6111242100992/perly-jaouda", + "brands": "Jaouda" + }, + { + "id": "off-nova3-04-6111246722039", + "expected_nova_group": 3, + "product_name": "fromage herbes milky food", + "ingredient_text": "Lait de vache pasteurisé, crème fraîche pasteurisée, ferments lactiques, présure, sel, ail et fines herbes.", + "source": "Open Food Facts", + "source_code": "6111246722039", + "source_url": "https://world.openfoodfacts.org/product/6111246722039/fromage-herbes-milky-food-milky-foof", + "brands": "milky foof" + }, + { + "id": "off-nova3-05-6111242103702", + "expected_nova_group": 3, + "product_name": "Yaourt Grec Muesli", + "ingredient_text": "Lait frais, crème, poudre lait, sucre, amidon, préparation de céréales et de fruits secs (amandes, noix, miel, raisins secs, blé, avoine, seigle), ferments lactiques, arômes Matière grasse: 7%", + "source": "Open Food Facts", + "source_code": "6111242103702", + "source_url": "https://world.openfoodfacts.org/product/6111242103702/yaourt-grec-muesli-jaouda-110g", + "brands": "Jaouda" + }, + { + "id": "off-nova3-06-6111266962187", + "expected_nova_group": 3, + "product_name": "Lait de la ferme jaouda", + "ingredient_text": "Lait frais pasteurisé demi écrémé à 15g/l de matière grasse", + "source": "Open Food Facts", + "source_code": "6111266962187", + "source_url": "https://world.openfoodfacts.org/product/6111266962187/lait-jaouda", + "brands": "Jaouda" + }, + { + "id": "off-nova3-07-6111246721278", + "expected_nova_group": 3, + "product_name": "Cream cheese", + "ingredient_text": "Lait de Vache pasteurisé, Crème fraiche pasteurisée ferments lactiques, Présure,Sel.", + "source": "Open Food Facts", + "source_code": "6111246721278", + "source_url": "https://world.openfoodfacts.org/product/6111246721278/cream-cheese-original", + "brands": "Original" + }, + { + "id": "off-nova3-08-6111242100206", + "expected_nova_group": 3, + "product_name": "Le nature", + "ingredient_text": "lait frais entier, poudre de lait écrémé, amidon, ferment lactique sélectionné", + "source": "Open Food Facts", + "source_code": "6111242100206", + "source_url": "https://world.openfoodfacts.org/product/6111242100206/le-nature-jaouda", + "brands": "Jaouda" + }, + { + "id": "off-nova3-09-6111031005064", + "expected_nova_group": 3, + "product_name": "Tonik", + "ingredient_text": "Coffret fourré au cacao (41,6%) et à la vanille (208) - Ingrédients Farine de blé, sucre, huile végétale non hydrogénée (huile de palme), filtrat de lait, poudre de cacao Émulsifiant à faible teneur en cacao (322) Lécithine de soja) Agent levant (5000) Sucre artificiel (vanilline) Sel Contient du lait, du blé (gluten) du soja", + "source": "Open Food Facts", + "source_code": "6111031005064", + "source_url": "https://world.openfoodfacts.org/product/6111031005064/tonik", + "brands": "tonik" + }, + { + "id": "off-nova3-10-6111162000716", + "expected_nova_group": 3, + "product_name": "Confiture d'abricot", + "ingredient_text": "Abricots, sucre, acide citrique", + "source": "Open Food Facts", + "source_code": "6111162000716", + "source_url": "https://world.openfoodfacts.org/product/6111162000716/confiture-d-abricot-delicia", + "brands": "Delicia" + }, + { + "id": "off-nova4-01-3175680011480", + "expected_nova_group": 4, + "product_name": "Sésame", + "ingredient_text": "Farine de blé 57%, sucre de canne roux, huile de colza, sésame toasté 10,6%, germe de blé 5,4%, farine complète de blé 5,4%, arôme naturel, magnésium, émulsifiant : lécithines, poudres à lever (tartrates de potassium, carbonates de sodium, carbonates d'ammonium), sel de mer, amidon de blé, vitamines (E, PP, B6, B1, B9). Fabriqué dans un atelier qui utilise des fruits à coque, du lait, du lupin, de la moutarde, des oeufs et du soja", + "source": "Open Food Facts", + "source_code": "3175680011480", + "source_url": "https://world.openfoodfacts.org/product/3175680011480/sesame-gerble", + "brands": "Gerblé" + }, + { + "id": "off-nova4-02-5449000000996", + "expected_nova_group": 4, + "product_name": "coca-cola", + "ingredient_text": "carbonated water, sugar, colour (caramel e150d), acid (phosphoric acid), natural flavourings, caffeine flavouring,", + "source": "Open Food Facts", + "source_code": "5449000000996", + "source_url": "https://world.openfoodfacts.org/product/5449000000996/coca-cola", + "brands": "Coca-Cola" + }, + { + "id": "off-nova4-03-6111246721261", + "expected_nova_group": 4, + "product_name": "Fromage Blanc Nature", + "ingredient_text": "Lait de vache pasteurisé, Protéines de lait de vache, Crème de lait de vache, Ferment lactique, Amidon, Epaississant, Sorbate de potassium", + "source": "Open Food Facts", + "source_code": "6111246721261", + "source_url": "https://world.openfoodfacts.org/product/6111246721261/fromage-blanc-nature-milky-food-professional", + "brands": "Milky Food Professional" + }, + { + "id": "off-nova4-04-6111242106949", + "expected_nova_group": 4, + "product_name": "Jben", + "ingredient_text": "lait frais entier, crème, stabilisants, amidon, carraghénane, sel, ferments lactiques matière grasse 21%", + "source": "Open Food Facts", + "source_code": "6111242106949", + "source_url": "https://world.openfoodfacts.org/product/6111242106949/jben-jaouda", + "brands": "Jaouda" + }, + { + "id": "off-nova4-05-3017620425035", + "expected_nova_group": 4, + "product_name": "Nutella", + "ingredient_text": "Sucre, huile de palme, NOISETTES 13%, LAIT écrémé en poudre 8,7%, cacao maigre 7,4%, émulsifiants: lécithines [SOJA]; vanilline. Sans gluten", + "source": "Open Food Facts", + "source_code": "3017620425035", + "source_url": "https://world.openfoodfacts.org/product/3017620425035/pate-a-tartiner-aux-noisettes-et-au-cacao-nutella", + "brands": "Nutella" + }, + { + "id": "off-nova4-06-3256540000698", + "expected_nova_group": 4, + "product_name": "Pains au lait", + "ingredient_text": "Farine de _blé_ 43%, Levain 21% (Farine de _blé_ 11%, Eau, Sel), Sucre, _Œufs_ frais 11%, _Beurre_ pâtissier, Huile de colza, Levure, _Lait_ écrémé en poudre (Équivalent à 10% de lait écrémé reconstitué), Émulsifiant: mono et diglycérides d'acides gras, Sel, Protéines de _blé_, _Gluten_ de blé, Extrait de carotte, Agent de traitement de la farine : acide ascorbique, Protéines de _lait_", + "source": "Open Food Facts", + "source_code": "3256540000698", + "source_url": "https://world.openfoodfacts.org/product/3256540000698/pains-au-lait-brioche-pasquier", + "brands": "Brioche Pasquier" + }, + { + "id": "off-nova4-07-7622210601988", + "expected_nova_group": 4, + "product_name": "Granola L'Original", + "ingredient_text": "Farine de BLÉ 36%, chocolat au LAIT 27 % [sucre, pâte de cacao, beurre de cacao, lactosérum en poudre (de LAIT), LAIT écrémé en poudre, graisses végétales (karité, palme en proportion variable), BEURRE concentré, émulsifiants (lécithines de SOJA, E476), lactose (de LAIT), arôme], huile de palme, farine complète de BLE 12 %, sucre, sirop de sucre, poudres à lever (carbonates de sodium, carbonates d'ammonium), sel, correcteur d'acidité (acide citrique). Peut contenir : fruits à coque.", + "source": "Open Food Facts", + "source_code": "7622210601988", + "source_url": "https://world.openfoodfacts.org/product/7622210601988/granola-l-original-lu", + "brands": "LU" + }, + { + "id": "off-nova4-08-7622210449283", + "expected_nova_group": 4, + "product_name": "Prince Goût Chocolat au blé complet", + "ingredient_text": "Céréale 50 % (Farine de blé 34,8 %, farine de blé complet 15,2 %), sucre, huiles végétales (palme, colza), cacao maigre en poudre 4,5 %, sirop de glucose, amidon de blé, poudres à lever (carbonates d'ammonium, carbonates de sodium), émulsifiant (lécithines de soja), sel, lait écrémé en poudre, perméat de lactosérum (de lait), arômes. Peut contenir œuf.", + "source": "Open Food Facts", + "source_code": "7622210449283", + "source_url": "https://world.openfoodfacts.org/product/7622210449283/prince-gout-chocolat-au-ble-complet-mondelez", + "brands": "Mondelez" + }, + { + "id": "off-nova4-09-6111242100305", + "expected_nova_group": 4, + "product_name": "jaouda Cremy", + "ingredient_text": "Lait frais, sucre, poudre de lait écrémé, texturant : amidon, crème, arôme, ferments lactiques sélectionnés, matière grasse 3%", + "source": "Open Food Facts", + "source_code": "6111242100305", + "source_url": "https://world.openfoodfacts.org/product/6111242100305/cremy-jaouda", + "brands": "jaouda" + }, + { + "id": "off-nova4-10-6111184004129", + "expected_nova_group": 4, + "product_name": "Mayonnaise recette originale", + "ingredient_text": "Huile de soja, eau, vinaigre de table, jaune d’œufs, moutarde", sel, stabilisants (E412, E415), conservateur : (E202, E211) et antioxydant : E330. * contient des sulfites", + "source": "Open Food Facts", + "source_code": "6111184004129", + "source_url": "https://world.openfoodfacts.org/product/6111184004129/mayonnaise-original-recipe-star", + "brands": "Star" + } +] diff --git a/backend/prompts/food_label_full_analysis_prompt.md b/backend/prompts/food_label_full_analysis_prompt.md index 1e9d1d9..822f11e 100644 --- a/backend/prompts/food_label_full_analysis_prompt.md +++ b/backend/prompts/food_label_full_analysis_prompt.md @@ -1,131 +1,86 @@ # Zest Full Food Label Analysis Contract -You are the backend-owned Zest food-label analysis engine. The Android app sends OCR text only; all instructions live here on the backend. +You are the backend-owned Zest food-label analysis engine. Android app sends OCR text only; all instructions live here on the backend. -Return exactly one JSON object matching the provided schema. No markdown, no prose outside JSON, no extra keys, no trailing commas. +Return exactly one JSON object matching the provided schema. No markdown, no prose outside JSON, no extra keys. Be fast: use the decision ladder below; do not debate alternatives. ## Inputs -The input JSON may contain: -- `rawIngredientText`: OCR or barcode ingredient text. -- `ingredients`: a rough split of the raw text. -- `productName`, `barcode`, `locale`: optional context only. +Use ingredient evidence only from `rawIngredientText` and `ingredients`. `productName`, `barcode`, and `locale` are weak context only. -Use only ingredient evidence from `rawIngredientText` and `ingredients`. Ignore package claims, marketing text, nutrition facts, preparation instructions, barcode numbers, storage text, manufacturer text, certifications, and advisory allergen statements. +Ignore marketing, nutrition facts, serving text, barcode-only text, storage/manufacturer/package text, certifications, and advisory allergens. -## Internal Order +## Sequence Perform this sequence silently inside this single model call: -1. Decide whether the text contains a consumable food item or meaningful food ingredient evidence. -2. If non-food, return `containsConsumableFoodItem=false`, `novaGroup=0`, a user-safe rejection reason, and empty ingredient/allergen lists. -3. If food, extract and clean ingredient names before doing marker or allergen work. -4. Detect ultra-processed markers only from the cleaned ingredient list. Each marker `name` must exactly match one value in `correctedIngredients`. -5. Detect allergens only from cleaned ingredients. Do not use "may contain", "made in a facility", "traces of", "free from", or other advisory statements. -6. Classify final NOVA group using cleaned ingredient evidence, visible industrial markers, and the NOVA rules below. +1. Food gate: decide whether the text contains food, beverage, or culinary ingredient evidence. +2. Cleaned ingredients: extract concise ingredient names. +3. Markers: identify only strong NOVA 4 markers. Each marker `name` must exactly match one value in `correctedIngredients`. +4. Allergens: detect only from Cleaned ingredients. +5. NOVA: apply the decision ladder. Stop at the first clearly matching rule. Dependency example: Raw text: "Ingredients: wheat flour, sugar, soy lecithin. May contain milk." Cleaned ingredients: ["Wheat Flour", "Sugar", "Soy Lecithin"] -Ultra-processed markers: [{"name":"Soy Lecithin","reason":"Emulsifier used in a formulation."}] +Ultra-processed markers: [{"name":"Soy Lecithin","reason":"Emulsifier."}] Allergens: ["Wheat", "Soy"] Do not add "Milk" because "may contain milk" is advisory text, not an ingredient. -## Non-Food Rule +## Food Gate -If the text is about a wall, room, object, document, barcode-only output, random scene, non-food product, or anything without consumable food ingredient evidence, return: +Food/beverage evidence includes water/mineral water, coffee, tea, milk, produce, grains, nuts, oils, salt, sugar, vinegar, spices, and named food ingredients. Do not reject mineral-water labels. -{ - "nova": { - "containsConsumableFoodItem": false, - "novaGroup": 0, - "summary": "Text doesn't contain any consumable food item.", - "rejectionReason": "Text doesn't contain any consumable food item.", - "confidence": 0.0, - "warnings": ["No food ingredient evidence was found in the supplied text."] - }, - "ingredients": { - "correctedIngredients": [], - "ultraProcessedIngredients": [], - "confidence": 0.0, - "warnings": [] - }, - "allergens": { - "allergens": [], - "confidence": 0.0, - "warnings": [] - } -} - -## Ingredient Cleanup - -Extract ingredient names only. Preserve order as much as possible. Correct obvious OCR errors conservatively. Remove labels such as "Ingredients:" and remove duplicated punctuation, bullets, quantities, nutrition facts, claims, advisory statements, serving text, and company/package text. - -Split comma, semicolon, line-break, bullet, and clear sub-ingredient lists. If a compound ingredient lists visible sub-ingredients, keep the compound name when useful and also include visible sub-ingredients. Do not invent missing ingredients or infer ingredients from product type, brand, flavor name, or assumptions. - -Use readable title casing such as "Sugar", "Wheat Flour", "Natural Flavor", "Soy Lecithin", "Modified Corn Starch". Keep parenthetical details only when part of the ingredient identity. Example: "Lecithin (Soy)" may become "Soy Lecithin". +If there is no consumable food/beverage/ingredient evidence, return `containsConsumableFoodItem=false`, `novaGroup=0`, empty ingredient/allergen lists, summary and rejectionReason "Text doesn't contain any consumable food item.", confidence `0.0`, and warning "No food ingredient evidence was found in the supplied text." -## Ultra-Processed Markers +## Clean Ingredients -`ultraProcessedIngredients` controls red ingredient capsules. Only include clear industrial or ultra-processed markers from `correctedIngredients`; every marker name must exactly match a cleaned ingredient. +Return title-cased ingredient names in order. Remove "Ingredients:", quantities when not needed, nutrition/claims, advisory allergen text, and package text. Split clear comma/semicolon/line-break/sub-ingredient lists. -Mark clear examples from these categories: -- Flavor systems: Natural Flavor, Artificial Flavor, Smoke Flavor, Vanillin. -- Color additives: Artificial Color, Caramel Color, Red 40, Yellow 5, Blue 1, Titanium Dioxide, Annatto Color. -- Non-sugar sweeteners: Aspartame, Sucralose, Acesulfame Potassium, Saccharin, Steviol Glycosides, Monk Fruit Extract when used as sweetener, Erythritol, Xylitol, Sorbitol, Maltitol. -- Emulsifiers: Soy Lecithin, Sunflower Lecithin in complex formulation, Lecithin in complex formulation, Mono- and Diglycerides, Polysorbates, DATEM, Sodium Stearoyl Lactylate, PGPR. -- Stabilizers/gums/thickeners: Xanthan Gum, Guar Gum, Gellan Gum, Cellulose Gum, Carboxymethylcellulose, Carrageenan, Locust Bean Gum, Acacia Gum, Microcrystalline Cellulose, Methylcellulose, Sodium Alginate. -- Modified starches and industrial carbohydrates: Modified Starch, Modified Corn Starch, Maltodextrin, Dextrin, Dextrose, Fructose, Glucose Syrup, Corn Syrup, Corn Syrup Solids, High Fructose Corn Syrup, Invert Sugar, Polydextrose. -- Protein isolates/hydrolyzed proteins: Soy Protein Isolate, Soy Protein Concentrate, Whey Protein Isolate, Milk Protein Isolate, Caseinate, Hydrolyzed Vegetable Protein, Textured Vegetable Protein. -- Hydrogenated or interesterified fats: Hydrogenated Oil, Partially Hydrogenated Oil, Interesterified Oil, hydrogenated shortening. -- Industrial preservatives and enhancers: Sodium Benzoate, Potassium Sorbate, Calcium Propionate, Sodium Nitrite, TBHQ, BHA, BHT, Monosodium Glutamate, Disodium Inosinate, Disodium Guanylate. -- Other clear anti-caking, anti-foaming, glazing, firming, bulking, texture, processing, or appearance agents. +Do not infer missing ingredients from product, brand, or flavor. Keep names concise, but include every ingredient needed for classification, allergens, and user-visible review. -Do not mark basic pantry ingredients red: sugar, honey, maple syrup, salt, vinegar, flour, rice flour, wheat flour, corn starch, potato starch, tapioca starch, olive oil, sunflower oil, canola oil, coconut oil, butter, ghee, milk, egg, cream, cocoa powder, spices, herbs, garlic powder, onion powder, vanilla extract. +## Strong NOVA 4 Markers -Do not mark allergens red unless the ingredient itself is also an ultra-processed marker. Do not mark every additive red automatically. If unsure, omit the marker and lower confidence if needed. Reasons must be short and specific. +Only mark these when present as ingredients: +- Flavors/colors: Natural Flavor, Artificial Flavor, Smoke Flavor, Caramel Color, Red 40, Yellow 5, Blue 1, Titanium Dioxide. +- Sweeteners/polyols: Aspartame, Sucralose, Acesulfame Potassium, Saccharin, Steviol Glycosides, Erythritol, Xylitol, Sorbitol, Maltitol. +- Emulsifiers/gums/stabilizers: Soy Lecithin in complex formulation, Mono- and Diglycerides, Polysorbates, DATEM, Sodium Stearoyl Lactylate, PGPR, Xanthan Gum, Guar Gum, Gellan Gum, Cellulose Gum, Carrageenan, Methylcellulose. +- Industrial fractions: Modified Starch, Modified Corn Starch, Maltodextrin, Dextrin, Dextrose, Glucose Syrup, Corn Syrup, High Fructose Corn Syrup, Invert Sugar, Polydextrose, soluble corn fiber, protein isolate/concentrate, hydrolyzed protein, textured protein. +- Preservatives/enhancers/industrial fats: Sodium Benzoate, Potassium Sorbate, Calcium Propionate, Sodium Nitrite, TBHQ, BHA, BHT, MSG, Disodium Inosinate, Disodium Guanylate, Hydrogenated Oil, Interesterified Oil. -## Allergen Detection +Weak items that should NOT trigger NOVA 4 by themselves: citric/ascorbic acid, acidity regulator, pectin, baking powder/soda, enriched flour, vitamins/minerals, iodine, iodized salt, anti-caking agent in salt, vanillin/ethylvanillin in vanilla sugar, cultures, enzymes, rennet, milk powder, whey in dairy foods, unmodified starch/flour. -Detect only common US/Western allergens explicitly present in `correctedIngredients`. Return canonical names only, in this fixed order when present: -["Milk", "Egg", "Wheat", "Barley", "Rye", "Soy", "Peanut", "Tree Nuts", "Fish", "Shellfish", "Sesame"] - -Signals: -- Milk: milk, skim milk, milk powder, cream, butter, ghee, cheese, yogurt, whey, casein, caseinate, lactose. -- Egg: egg, egg white, egg yolk, dried egg, albumin, ovalbumin, lysozyme when egg-derived or in egg context. -- Wheat: wheat, wheat flour, whole wheat, wheat starch, wheat gluten, durum, semolina, farina, spelt, farro, einkorn, emmer, kamut, couscous when wheat-based, bulgur, atta, maida. -- Barley: barley, barley flour, barley malt, malt, malt extract, malt syrup. -- Rye: rye, rye flour, whole rye, rye meal, rye flakes. -- Soy: soy, soya, soybean, soy flour, soy protein, tofu, tempeh, edamame, miso, soy sauce, tamari, shoyu, soy lecithin, hydrolyzed soy protein. -- Peanut: peanut, peanuts, peanut flour, peanut protein, peanut butter, groundnut, arachis oil. -- Tree Nuts: almond, hazelnut, walnut, cashew, pistachio, pecan, macadamia, brazil nut, pine nut, chestnut, coconut, almond extract, marzipan, praline when nut-based. -- Fish: fish, anchovy, tuna, salmon, cod, haddock, pollock, sardine, mackerel, trout, bonito, fish sauce, fish gelatin, isinglass. -- Shellfish: shrimp, prawn, crab, lobster, crayfish, crawfish, krill, clam, oyster, mussel, scallop, squid, octopus, cuttlefish, abalone. -- Sesame: sesame, sesame seed, sesame oil, sesame paste, tahini, benne, gingelly, til. - -Do not infer allergens from product type, cuisine, brand, generic "flour", generic "starch", generic "lecithin", "natural flavor", "nutty flavor", "seafood", "omega-3", or advisory statements. +Do not mark pantry foods: sugar, honey, salt, vinegar, flour, starch, oils, butter, milk, egg, cream, cocoa, spices, herbs, vanilla extract. If unsure, omit marker. -## NOVA Classification +## NOVA Decision Ladder -Choose exactly one overall NOVA group using the highest group clearly supported by visible ingredient evidence. Do not average ingredients. Do not classify from brand, marketing claims, or assumptions. +Use first clearly matching rule: -NOVA 4: Assign for industrial formulations with clear ultra-processing markers such as flavors, non-sugar sweeteners, emulsifiers, stabilizers/gums, colorants, flavor enhancers, modified starches, hydrogenated/interesterified oils, hydrolyzed proteins, protein isolates, industrial sugars, refined carbohydrate fractions, reconstituted/mechanically separated animal ingredients, or complex refined formulations of starch/flour, sugar, oil, salt, and additives. One clear strong marker can be enough. +1. NOVA 4 if strong NOVA 4 marker exists OR the list is a clear industrial formulation dominated by refined fractions/additives. Long list alone is not NOVA 4. +2. NOVA 2 if product is mainly a culinary ingredient: oil, butter, spreadable fat, sugar, vanilla sugar, salt including iodized/anti-caking salt, honey, syrup, vinegar, starch, flour, nut/peanut butter made mainly of nuts/oil/salt/sugar. +3. NOVA 1 if ingredients are only minimally processed food/drink: water/mineral water, coffee/tea, milk, plain yogurt, eggs, fruit, vegetables, grains, legumes, nuts/seeds, meat/fish, chopped tomatoes/tomato juice with only citric acid/acidity regulator. +4. NOVA 3 if recognizable foods are combined with salt/sugar/oil/vinegar/cultures/enzymes/simple preservation and no strong NOVA 4 marker: bread, cheese, cream cheese, canned foods, salted nuts, jam, ketchup, pickles, sweetened/plain dairy, simple bakery foods. +5. Ambiguous adjacent groups: choose the lower group unless strong evidence supports the higher group; lower confidence and add one short warning. -NOVA 3: Assign for relatively simple foods made by combining recognizable foods with salt, sugar, oil, vinegar, or other culinary ingredients, with no clear NOVA 4 markers. Examples: simple bread, cheese, canned vegetables, salted nuts, fruits in syrup, simple pickles, simple jams. +Foreign-language labels: classify from clear ingredient meaning; do not guess higher. -NOVA 2: Assign only when the product itself is primarily a culinary ingredient: sugar, salt, honey, vinegar, starch, butter, edible oils, syrups, flours presented as culinary ingredients. +## Allergens -NOVA 1: Assign when visible ingredients are only unprocessed or minimally processed foods with no added culinary ingredients or additives: fruits, vegetables, grains, legumes, meat, fish, eggs, milk, plain yogurt, nuts, seeds, plain spices, water. - -Tie-breakers: clear NOVA 4 marker means Group 4. Recognizable food plus salt/sugar/oil with no NOVA 4 marker means Group 3. Culinary ingredient alone means Group 2. Minimally processed only means Group 1. If adjacent groups are ambiguous, choose the higher group only when evidence supports it. Never default to Group 4 because the list is long. +Return explicit allergens only, ordered as: +["Milk", "Egg", "Wheat", "Barley", "Rye", "Soy", "Peanut", "Tree Nuts", "Fish", "Shellfish", "Sesame"] -## Confidence, Warnings, Summary +Signals: milk/cream/butter/cheese/yogurt/whey/casein/lactose; egg; wheat/barley/rye/flour/malt/semolina/durum/spelt/couscous/bulgur; soy/soya/soy lecithin; peanut; almond/hazelnut/walnut/cashew/pistachio/pecan/macadamia/coconut; named fish/shellfish; sesame/tahini. -Confidence: 0.90-1.00 for clear evidence, 0.75-0.89 for good evidence with minor ambiguity/noise, 0.55-0.74 for incomplete or somewhat uncertain evidence, 0.30-0.54 for noisy partial evidence, below 0.30 for very poor evidence. +Do not infer allergens from product type, brand, generic flour/starch/lecithin/flavor, or advisory text such as "may contain", "traces of", or "made in a facility". -Warnings should mention only incomplete evidence, ambiguous evidence, or uncertainty. Do not mention image analysis, brand, package claims, or medical advice. +## Output Limits -Summary must be under 50 words, warm, professional, and useful for shopping. Mention the main processing reason from ingredient evidence. For non-food, use the non-food summary. Do not overstate safety or give medical advice. +- `summary`: one sentence under 18 words. +- `correctedIngredients`: include all meaningful cleaned ingredients; avoid duplicates and non-ingredient text. +- `ultraProcessedIngredients`: include every strong marker found in `correctedIngredients`; do not cap or omit real markers. +- `warnings`: include only useful ambiguity or evidence-quality warnings; avoid boilerplate. +- Use empty arrays when none. +- Confidence: 0.90+ clear; 0.75-0.89 minor ambiguity; 0.55-0.74 noisy/partial; lower for poor evidence. ## Output Shape diff --git a/backend/tests/test_app.py b/backend/tests/test_app.py index 4e46af8..ada4b3a 100644 --- a/backend/tests/test_app.py +++ b/backend/tests/test_app.py @@ -50,6 +50,23 @@ def test_healthz(): assert r.json() == {"status": "ok"} +def test_version_endpoint_returns_module_manifest(): + r = client.get("/version") + assert r.status_code == 200 + body = r.json() + assert body["schemaVersion"] == 1 + assert body["releaseVersion"] == main.SERVICE_VERSION + assert main.app.version == body["releaseVersion"] + modules = {module["id"]: module for module in body["modules"]} + assert modules["android-app"]["path"] == ":app" + assert modules["backend-proxy"]["path"] == "backend" + assert modules["backend-prompts"]["path"] == "backend/prompts" + serialized = json.dumps(body) + assert "ingredient_text" not in serialized + assert "rawIngredientText" not in serialized + assert "Zest Full Food Label Analysis Contract" not in serialized + + def test_openapi_docs_disabled_by_default(): assert client.get("/docs").status_code == 404 assert client.get("/openapi.json").status_code == 404 @@ -63,6 +80,14 @@ def test_full_analysis_prompt_is_backend_owned_and_sequential(): assert "Cleaned ingredients" in prompt assert "Do not add \"Milk\" because \"may contain milk\" is advisory text" in prompt assert "exactly match one value in `correctedIngredients`" in prompt + assert "Do not reject mineral-water labels" in prompt + assert "Weak items that should NOT trigger NOVA 4 by themselves" in prompt + assert "Long list alone is not NOVA 4" in prompt + assert "NOVA Decision Ladder" in prompt + assert "NOVA 2 if product is mainly a culinary ingredient" in prompt + assert "NOVA 3 if recognizable foods are combined" in prompt + assert "include every strong marker found" in prompt + assert "do not cap or omit real markers" in prompt assert not (PROMPT_DIR / "food_label_classification_prompt.md").exists() assert not (PROMPT_DIR / "food_label_ingredient_analysis_prompt.md").exists() @@ -262,7 +287,7 @@ class Client: assert "Zest Full Food Label Analysis Contract" in calls[0]["contents"] assert "wheat flour" in calls[0]["contents"] config = calls[0]["config"].model_dump(by_alias=True, exclude_none=True) - assert config["maxOutputTokens"] == 700 + assert config["maxOutputTokens"] == main.FULL_ANALYSIS_MAX_OUTPUT_TOKENS assert config["responseMimeType"] == "application/json" assert config["responseSchema"] is main.FullAnalysisSchema diff --git a/backend/tests/test_benchmark_analyze.py b/backend/tests/test_benchmark_analyze.py new file mode 100644 index 0000000..307c5ab --- /dev/null +++ b/backend/tests/test_benchmark_analyze.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import json +from collections import Counter + +import benchmark_analyze + + +def test_nova_benchmark_corpus_has_ten_products_per_group(): + corpus = benchmark_analyze.load_corpus() + + assert len(corpus) == 40 + counts = Counter(sample["expected_nova_group"] for sample in corpus) + assert counts == {1: 10, 2: 10, 3: 10, 4: 10} + assert len({sample["id"] for sample in corpus}) == 40 + assert all(sample["product_name"].strip() for sample in corpus) + assert all(sample["ingredient_text"].strip() for sample in corpus) + + +def test_benchmark_reports_latency_and_nova_accuracy(): + corpus = [ + { + "id": "nova1-oats", + "product_name": "Oats", + "ingredient_text": "whole grain oats", + "expected_nova_group": 1, + }, + { + "id": "nova4-cereal", + "product_name": "Cereal", + "ingredient_text": "corn syrup, soy lecithin, natural flavor", + "expected_nova_group": 4, + }, + ] + responses = { + "Oats": {"nova": {"novaGroup": 1}, "usage": {"inputTokens": 10, "outputTokens": 4, "totalTokens": 20}}, + "Cereal": {"nova": {"novaGroup": 3}, "usage": {"inputTokens": 12, "outputTokens": 5, "totalTokens": 25}}, + } + + def fake_post(_url, payload, _timeout): + return 200, json.dumps(responses[payload["product_name"]]) + + result = benchmark_analyze.run_benchmark( + analyze_url="https://proxy.test/analyze", + corpus=corpus, + iterations=1, + timeout=1, + post_fn=fake_post, + ) + + assert result["totalRequests"] == 2 + assert result["successfulRequests"] == 2 + assert result["novaCorrect"] == 1 + assert result["novaAccuracy"] == 0.5 + assert result["errors"] == {"nova_mismatch": 1} + assert result["byExpectedNovaGroup"]["1"]["correct"] == 1 + assert result["byExpectedNovaGroup"]["4"]["correct"] == 0 + assert result["results"][0]["estimatedHiddenThinkingTokens"] == 6 diff --git a/documentation/07-testing-release.md b/documentation/07-testing-release.md index ae5a437..6d5c4de 100644 --- a/documentation/07-testing-release.md +++ b/documentation/07-testing-release.md @@ -23,6 +23,7 @@ Run this after most code changes: - retired demo, sample, or rule-based classifier files reappearing, - macOS dataless placeholders under `app/src` that can make Gradle or KSP appear stuck. +- missing module version entries or stale module version documentation. ## Unit Tests @@ -91,6 +92,7 @@ flowchart TB - Unit tests pass. - `verifySourceTreeForBuild` passes. +- `backend/module_versions.json` contains every Gradle module, top-level Android package module, backend service, and backend prompt contract. - Release APK assembles with signing environment variables present. - Android test APK assembles. - `rg "BuildConfig|local.properties|USDA_API_KEY" app/src/main app/build.gradle.kts` shows no embedded key source. diff --git a/documentation/12-module-versioning.md b/documentation/12-module-versioning.md new file mode 100644 index 0000000..590e2c4 --- /dev/null +++ b/documentation/12-module-versioning.md @@ -0,0 +1,59 @@ +# Module Versioning + +Zest tracks module versions through one machine-readable manifest: + +```text +backend/module_versions.json +``` + +That file is the production source of truth. It is packaged with the Cloud Run backend, exposed through `GET /version`, and checked by the Android release guard so new source modules cannot be added without an explicit version entry. + +## Current Module Versions + +| Module ID | Kind | Path | Version | Runtime Surface | +| --- | --- | --- | --- | --- | +| `android-app` | android-gradle-module | `:app` | `1.0.1` | Android `BuildConfig.VERSION_NAME` | +| `backend-proxy` | cloud-run-service | `backend` | `1.0.1` | FastAPI `app.version` and `GET /version` | +| `analysis` | android-package | `app/src/main/java/com/b2/ultraprocessed/analysis` | `1.0.1` | module version manifest | +| `barcode` | android-package | `app/src/main/java/com/b2/ultraprocessed/barcode` | `1.0.0` | module version manifest | +| `camera` | android-package | `app/src/main/java/com/b2/ultraprocessed/camera` | `1.0.0` | module version manifest | +| `classify` | android-package | `app/src/main/java/com/b2/ultraprocessed/classify` | `1.0.0` | module version manifest | +| `ingredients` | android-package | `app/src/main/java/com/b2/ultraprocessed/ingredients` | `1.0.0` | module version manifest | +| `network` | android-package | `app/src/main/java/com/b2/ultraprocessed/network` | `1.0.0` | module version manifest | +| `ocr` | android-package | `app/src/main/java/com/b2/ultraprocessed/ocr` | `1.0.0` | module version manifest | +| `storage` | android-package | `app/src/main/java/com/b2/ultraprocessed/storage` | `1.0.0` | module version manifest | +| `ui` | android-package | `app/src/main/java/com/b2/ultraprocessed/ui` | `1.0.1` | module version manifest | +| `backend-prompts` | backend-contract-assets | `backend/prompts` | `1.0.3` | backend-owned prompt files | + +## Production Tracking Contract + +- Every Gradle module must have one manifest entry with `kind = android-gradle-module`. +- Every top-level Android package under `app/src/main/java/com/b2/ultraprocessed` must have one manifest entry with `kind = android-package`. +- Every deployable backend service must have one manifest entry with a runtime endpoint that can report the deployed version. +- Backend prompt contracts are versioned separately from the backend service because prompt changes can alter model behavior without changing API transport. +- Documentation must include each manifest module ID and version. + +## Verification + +Android `verifySourceTreeForBuild` runs `verifyModuleVersionManifest`. The task fails when: + +- `backend/module_versions.json` is missing or invalid, +- a top-level Android package exists without a version entry, +- a version entry points to a missing path, +- a Gradle module exists without a manifest entry, +- documentation is missing a module ID or version from the manifest. + +Backend tests verify: + +- FastAPI `app.version` matches `releaseVersion`, +- `GET /version` returns the manifest, +- `/version` does not expose request data, prompt bodies, OCR text, or model responses. + +## Version Bump Rules + +- Bump `android-app` when changing app behavior, UI, API contract handling, permissions, storage, release config, or bundled assets. +- Bump `backend-proxy` when changing request validation, endpoint behavior, model configuration, response contracts, deployment config, auth, or observability. +- Bump `backend-prompts` when changing prompt instructions, schemas, examples, model-output expectations, or chat safety behavior. +- Bump an Android package module when changing public behavior owned by that package. + +All module versions use semantic versioning. For coordinated releases, `releaseVersion` should match the app version shipped to users. Individual modules can move faster only when a change is internal and does not require a full app release. diff --git a/documentation/README.md b/documentation/README.md index b9ac90f..04f1e22 100644 --- a/documentation/README.md +++ b/documentation/README.md @@ -19,6 +19,8 @@ If you are not an Android developer, start with [00-android-app-guide.md](00-and - [09-todo-roadmap.md](09-todo-roadmap.md) - engineering and product backlog, including centralized navigation stack work for v2. - [10-responsible-ai-review.md](10-responsible-ai-review.md) - concise review of harmful gender, language, geographic, and socioeconomic behavior. - [11-maintenance-plan.md](11-maintenance-plan.md) - post-launch ownership, cadence, incident handling, patch policy, rollback, and support operations. +- [12-module-versioning.md](12-module-versioning.md) - module version manifest, production tracking contract, and verification rules. +- [performance_benchmark.md](performance_benchmark.md) - deployed backend response-time measurements, bottleneck analysis, and latency improvement plan. ## Current Product Contract diff --git a/documentation/performance_benchmark.md b/documentation/performance_benchmark.md new file mode 100644 index 0000000..6950308 --- /dev/null +++ b/documentation/performance_benchmark.md @@ -0,0 +1,310 @@ +# Performance Benchmark + +This document captures the current deployed backend response-time profile for Zest's model-backed calls and the likely causes of latency. The measurements below used synthetic ingredient text only; no human scan data, image data, OCR output, prompts, or model responses were stored. + +## Tested Service + +Base URL: + +```text +https://ultraprocessed-ai-proxy-894254677159.us-east1.run.app +``` + +Working endpoints: + +- `POST /analyze` +- `POST /chat` + +Observed note: + +- `GET /healthz` returned `404` on the deployed service during this benchmark. `/analyze` and `/chat` worked, so the service is reachable, but the deployed revision should expose a health route before production monitoring depends on it. + +## Benchmark Summary + +### Endpoint-Level Timings + +| Endpoint | Sample Count | Observed Latency | +| --- | ---: | --- | +| `/analyze` | 3 quick samples | `4.70s` to `4.90s` | +| `/chat` | 3 quick samples | `2.64s` to `2.94s` | +| `/chat` simpler question | 1 sample | `1.30s` | + +Benchmark run against `/analyze`: + +```json +{ + "totalRequests": 10, + "successfulRequests": 10, + "successRate": 1.0, + "p50Ms": 4939.31, + "p95Ms": 7341.21, + "p99Ms": 7341.21, + "errors": {} +} +``` + +### Analysis Samples + +| Sample | Payload Chars | Latency | Input Tokens | Output Tokens | Estimated Hidden/Thinking Tokens | NOVA Group | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Simple oats | 90 | `3.88s` | 2817 | 160 | 457 | 1 | +| Cereal | 188 | `5.03s` | 2859 | 336 | 502 | 4 | +| Noisy OCR | 221 | `7.61s` | 2892 | 254 | 988 | 4 | +| Allergen dense | 195 | `6.22s` | 2865 | 323 | 666 | 4 | +| Non-food | 142 | `2.64s` | 2853 | 167 | 198 | 0 | + +### 40-Product NOVA Corpus + +The corpus in `backend/nova_benchmark_corpus.json` uses Open Food Facts products with populated +`nova_group` and ingredient text. Each row includes the Open Food Facts product code and product URL. +The expected NOVA value is the Open Food Facts `nova_group` value for that product. + +Live benchmark command: + +```bash +.venv/bin/python backend/benchmark_analyze.py \ + https://ultraprocessed-ai-proxy-894254677159.us-east1.run.app \ + --iterations 1 \ + --timeout 90 \ + --details +``` + +Result: + +```json +{ + "corpusSize": 40, + "totalRequests": 40, + "successfulRequests": 40, + "successRate": 1.0, + "novaCorrect": 28, + "novaAccuracy": 0.7, + "p50Ms": 7727.49, + "p95Ms": 14551.48, + "p99Ms": 18015.0, + "errors": { + "nova_mismatch": 12 + } +} +``` + +By expected NOVA group: + +| Expected Group | Samples | Correct | Accuracy | p50 | p95 | +| --- | ---: | ---: | ---: | ---: | ---: | +| NOVA 1 | 10 | 8 | `0.8` | `6684.11ms` | `14551.48ms` | +| NOVA 2 | 10 | 6 | `0.6` | `5425.98ms` | `9553.46ms` | +| NOVA 3 | 10 | 4 | `0.4` | `7943.25ms` | `14534.81ms` | +| NOVA 4 | 10 | 10 | `1.0` | `9057.90ms` | `18015.00ms` | + +The real-product corpus is harder than the prior synthetic corpus. It includes multilingual ingredient +text, noisy ingredient formatting, and Open Food Facts labels that may reflect database-specific +classification choices. Treat mismatches as review cases, not automatically as model defects. NOVA 4 +remains the slowest group because those examples generally produce larger outputs and more +hidden/thinking tokens. + +Prompt optimization `backend-prompts` version `1.0.3` targets these benchmark findings: + +- keep mineral-water labels classified as food/beverage instead of non-food, +- prevent weak ingredients such as citric acid, enrichment, anti-caking agents, cultures, enzymes, and unmodified starch from forcing NOVA 4, +- handle culinary ingredient products such as oils, salt, sugar, butter, nut butter, and spreadable fats before promoting to NOVA 3/4, +- treat simple processed foods such as bread, cheese, cream cheese, canned foods, jams, ketchup, and sweetened dairy as NOVA 3 unless strong NOVA 4 markers are present, +- use a first-match NOVA decision ladder for easier Gemini 3.5 Flash classification, +- keep output concise without artificial caps that would suppress real ingredients or ultra-processed markers. + +The optimized prompt has not been measured on Cloud Run in this document yet. Local direct Vertex +testing was blocked by missing Application Default Credentials. Rerun the 40-product benchmark after +deploying the updated backend prompt. + +### Chat Samples + +| Question | Latency | Input Tokens | Output Tokens | Estimated Hidden/Thinking Tokens | +| --- | ---: | ---: | ---: | ---: | +| Which ingredient is most concerning? | `2.87s` | 543 | 70 | 239 | +| Why is this NOVA 4? | `2.62s` | 544 | 77 | 169 | +| Are there allergens? | `1.30s` | 541 | 29 | 68 | + +## Interpretation + +The slower path is not network transfer or Cloud Run routing. Time-to-first-byte and total request time were almost identical during testing, which means most latency occurs while the backend waits for the model response. + +The `/analyze` endpoint is slower than `/chat` because it performs the full structured analysis in one Gemini call: + +- food/non-food gate, +- cleaned ingredient list, +- ultra-processed marker detection, +- allergen detection, +- NOVA classification, +- short summaries and warnings. + +Even tiny ingredient payloads produce about `2.8k` input tokens for `/analyze`, so the backend analysis prompt dominates the request. Chat uses about `540` input tokens for the tested questions, so it has a much smaller model workload. + +The slowest `/analyze` samples also show higher estimated hidden/thinking tokens. The noisy OCR sample took `7.61s` and used about `988` hidden/thinking tokens, while the non-food sample took `2.64s` and used about `198`. + +## Current Bottleneck + +Primary bottleneck: + +```text +/analyze -> Gemini full-analysis generation +``` + +The likely contributors are: + +- long backend analysis prompt, +- structured JSON output requirement, +- multi-part reasoning in one call, +- hidden/thinking token budget, +- larger output for full ingredient/allergen/NOVA responses. + +The current user-visible "NOVA classification" wait time is therefore the full analysis wait time, not only a standalone NOVA group classification. + +## Recommended Backend Changes + +### 1. Add Explicit Thinking Budget + +Set Gemini thinking behavior explicitly for `/analyze`. + +Candidate defaults: + +```text +GEMINI_ANALYSIS_THINKING_BUDGET=0 +GEMINI_CHAT_THINKING_BUDGET=0 +``` + +If quality drops or the model rejects zero for the selected model, test small fixed budgets: + +```text +128 +256 +``` + +Acceptance criteria: + +- `/analyze` p95 improves against the benchmark corpus. +- JSON validity remains at `100%` for the synthetic corpus. +- NOVA group, allergen, and ultra-processed marker quality does not regress materially in manual review. + +### 2. Add Privacy-Safe Timing Instrumentation + +Expose backend timing without logging human data. + +Recommended fields: + +- `requestTotalMs` +- `inputNormalizeMs` +- `promptBuildMs` +- `modelCallMs` +- `jsonCoerceMs` +- `inputTokens` +- `outputTokens` +- `totalTokens` +- `estimatedHiddenThinkingTokens` + +Do not log or persist: + +- OCR text, +- ingredient text, +- prompts, +- model responses, +- chat questions, +- image paths, +- user identifiers. + +Acceptance criteria: + +- Engineers can tell whether latency is inside the model call or backend processing. +- Production logs remain free of human scan data and model payloads. +- Timing metadata is optional and can be disabled or hidden from the app UI. + +### 3. Keep `/analyze` And `/chat` Tuned Separately + +The split endpoints are correct because they have different latency, prompt, output, and safety requirements. + +`/analyze` should optimize for: + +- strict schema validity, +- ingredient and allergen completeness, +- NOVA correctness, +- bounded output. + +`/chat` should optimize for: + +- short answers, +- current-result scope, +- session-only chat history, +- prompt-injection resistance, +- low latency. + +Acceptance criteria: + +- `/chat` changes cannot accidentally expand `/analyze` prompt or output size. +- `/analyze` changes cannot weaken chat scoping. +- Both endpoints have independent prompt files and config budgets. + +### 4. Revisit Prompt Size If p95 Remains Above 5s + +If explicit thinking budget does not bring `/analyze` p95 below the target, the next lever is reducing prompt and output size. + +Options: + +- shorten examples in the full-analysis prompt, +- move repeated guidance into compact schema descriptions, +- cap per-field summaries more aggressively, +- return fewer low-value warnings, +- keep only user-visible fields in the response. + +Acceptance criteria: + +- `/analyze` input tokens drop meaningfully from the current `~2.8k`. +- p95 drops below the product SLO or the product explicitly accepts the tradeoff. +- The prompt remains backend-owned; Android must not send prompts. + +### 5. Consider Progressive UX Only If Strict p95 Is Required + +If the product requires visible NOVA classification under `5s` at p95, one single full-analysis call may not be enough under all OCR conditions. + +An alternate UX can return a fast classification first and fill ingredient/allergen details later. This should be treated as a product decision because it changes the response lifecycle and visible UI behavior. + +Acceptance criteria: + +- The app clearly separates "classification ready" from "full analysis ready." +- No partial result is persisted beyond the active session. +- Backend prompts remain server-owned. + +## Follow-Up Test Cases + +### Latency + +- Run at least 30 requests against `/analyze` using `backend/nova_benchmark_corpus.json`. +- Run at least 30 requests against `/chat` using representative result questions. +- Track p50, p90, p95, p99, success rate, and HTTP error codes. +- Compare cold-ish and warm Cloud Run behavior separately when possible. + +### Correctness + +- Use at least 10 synthetic products per expected NOVA group. +- Simple minimally processed food returns NOVA 1 or 3 where appropriate. +- Ultra-processed cereal returns NOVA 4 and identifies industrial markers. +- Noisy OCR still produces valid JSON and bounded warnings. +- Non-food input returns `containsConsumableFoodItem=false`. +- Allergen-dense input detects common allergens without merging them into ingredient coloring. + +### Contract + +- `/analyze` returns only the existing app contract fields plus optional timing metadata. +- `/chat` returns only the chat contract. +- Android does not send prompts, schemas, API keys, or model configuration. +- Backend prompt changes do not require an Android app release unless the response contract changes. + +### Privacy And Logging + +- No request body appears in Cloud Run logs. +- No prompt text appears in Cloud Run logs. +- No model response appears in Cloud Run logs. +- No chat question appears in Cloud Run logs. +- Timing and token metadata are safe to log because they do not contain human scan content. + +## Current Recommendation + +The first implementation step should be explicit Gemini thinking-budget control plus privacy-safe timing instrumentation. That directly targets the observed latency source while preserving the current backend-owned prompt architecture and the one-call `/analyze` contract.