Qualify E5-Mistral-7B, and support multiple reference sets

benchmark-results/embedding/e5-mistral-7b-instruct-q4_k_m.json

@@ -0,0 +1,67 @@
+{
+  "workload" : "oracle-equivalence-v1",
+  "model" : "E5-Mistral-7B-Instruct Q4_K_M",
+  "backend" : "pure-java",
+  "artifactSha256" : "96f5b8305d2091b48dc1191c082e61cca1f0fc7181650106bed3dd5a495ab5e4",
+  "artifactSizeBytes" : 4368439520,
+  "probeSetSha256" : "a25952a4cb80e57c099291dda9b44ad2b713c0688c386015a1e0000bf6fbf8b9",
+  "probes" : 8,
+  "embeddingDimension" : 4096,
+  "pooling" : "last-token",
+  "normalized" : true,
+  "oracleBackend" : "llama.cpp",
+  "oracleVersion" : "6ea215d17",
+  "minimumOracleCosine" : 0.9995117362682604,
+  "meanOracleCosine" : 0.9996140137272909,
+  "maxComponentDelta" : 0.004982158541679382,
+  "maxNormDeviation" : 5.542114434042844E-9,
+  "minimumOracleCosineFloor" : 0.999,
+  "maxNormDeviationFloor" : 0.001,
+  "normalizationHeld" : true,
+  "qualified" : true,
+  "perProbe" : [ {
+    "probe" : "The cat sat on the mat.",
+    "cosine" : 0.9996254135885395,
+    "embedMillis" : 2574.312301
+  }, {
+    "probe" : "How do I reset my password?",
+    "cosine" : 0.9996566332252969,
+    "embedMillis" : 2493.382404
+  }, {
+    "probe" : "Quantum chromodynamics describes the strong interaction.",
+    "cosine" : 0.9995748110871204,
+    "embedMillis" : 3950.757927
+  }, {
+    "probe" : "a",
+    "cosine" : 0.9996833646648101,
+    "embedMillis" : 666.914917
+  }, {
+    "probe" : "The quick brown fox jumps over the lazy dog near the riverbank at dawn while birds sing overhead.",
+    "cosine" : 0.9995117362682604,
+    "embedMillis" : 8140.317039
+  }, {
+    "probe" : "Database backups are retained for 30 days.",
+    "cosine" : 0.9997086208948631,
+    "embedMillis" : 3919.615697
+  }, {
+    "probe" : "def add(a, b): return a + b",
+    "cosine" : 0.9995684641736909,
+    "embedMillis" : 4287.585216
+  }, {
+    "probe" : "¿Dónde está la biblioteca?",
+    "cosine" : 0.9995830659157456,
+    "embedMillis" : 3839.16599
+  } ],
+  "environment" : {
+    "host" : "rockhopper",
+    "osName" : "Linux",
+    "osVersion" : "7.0.11-76070011-generic",
+    "architecture" : "amd64",
+    "cpuModel" : "Intel(R) Core(TM) i9-14900HX",
+    "processors" : 32,
+    "physicalMemoryBytes" : 101073932288,
+    "javaVersion" : "25.0.3",
+    "javaVendor" : "Azul Systems, Inc.",
+    "vmName" : "OpenJDK 64-Bit Server VM"
+  }
+}

models-bench/src/main/java/com/integrallis/models/bench/EmbeddingEquivalence.java

@@ -24,6 +24,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * Compares embeddings produced here against vectors a reference implementation produced from the
@@ -62,8 +63,7 @@ public final class EmbeddingEquivalence {
 
   private static final String RESOURCE_ROOT = "/embedding-equivalence/";
   private static final String PROBES_RESOURCE = RESOURCE_ROOT + "probes.txt";
-  private static final String REFERENCE_RESOURCE =
-      RESOURCE_ROOT + "reference-qwen3-embedding-0.6b-q8_0.json";
+  private static final String REFERENCE_INDEX = RESOURCE_ROOT + "references.txt";
 
   private EmbeddingEquivalence() {}
 
@@ -266,16 +266,65 @@ public static String loadProbesRaw() {
   }
 
   /**
-   * Loads the committed reference vectors.
+   * Lists every committed reference set, in index order.
    *
-   * @return the pinned reference
+   * @return reference resource names
+   */
+  public static List<String> referenceNames() {
+    return readResource(REFERENCE_INDEX)
+        .lines()
+        .map(String::trim)
+        .filter(line -> !line.isEmpty() && !line.startsWith("#"))
+        .toList();
+  }
+
+  /**
+   * Finds the reference generated from an exact model artifact.
+   *
+   * <p>Selecting by digest rather than by name means the gate cannot compare a model against
+   * another model's vectors.
+   *
+   * @param artifactSha256 SHA-256 digest of a model artifact
+   * @return the matching reference, when one is committed
+   */
+  public static Optional<Reference> referenceFor(String artifactSha256) {
+    if (artifactSha256 == null || artifactSha256.isBlank()) {
+      return Optional.empty();
+    }
+    for (String name : referenceNames()) {
+      Reference reference = loadReference(name);
+      if (reference.artifactSha256().equalsIgnoreCase(artifactSha256)) {
+        return Optional.of(reference);
+      }
+    }
+    return Optional.empty();
+  }
+
+  /**
+   * Loads the first committed reference set.
+   *
+   * @return the first reference in the index
    */
   public static Reference loadReference() {
+    List<String> names = referenceNames();
+    if (names.isEmpty()) {
+      throw new IllegalStateException("no reference sets are committed");
+    }
+    return loadReference(names.get(0));
+  }
+
+  /**
+   * Loads one committed reference set by resource name.
+   *
+   * @param name file name under the embedding-equivalence resource directory
+   * @return the pinned reference
+   */
+  public static Reference loadReference(String name) {
     JsonNode root;
     try {
-      root = new ObjectMapper().readTree(readResource(REFERENCE_RESOURCE));
+      root = new ObjectMapper().readTree(readResource(RESOURCE_ROOT + name));
     } catch (IOException failure) {
-      throw new UncheckedIOException("reference vectors are not valid JSON", failure);
+      throw new UncheckedIOException("reference vectors are not valid JSON: " + name, failure);
     }
     JsonNode rows = root.get("vectors");
     List<float[]> vectors = new ArrayList<>(rows.size());

models-bench/src/main/java/com/integrallis/models/bench/EmbeddingEquivalenceCli.java

@@ -82,28 +82,28 @@ public static int run(String[] args) throws IOException {
       return usage("artifact does not exist: " + artifact);
     }
 
-    EmbeddingEquivalence.Reference reference = EmbeddingEquivalence.loadReference();
     List<String> probes = EmbeddingEquivalence.loadProbes();
 
-    // Both digests must hold or the comparison is against vectors from different inputs. Checking
-    // them is the difference between proving equivalence and proving nothing at all.
+    // The reference is chosen by artifact digest, so a model can only ever be compared against
+    // vectors generated from its own exact bytes.
+    String artifactSha256 = Hashing.sha256(artifact);
+    EmbeddingEquivalence.Reference reference =
+        EmbeddingEquivalence.referenceFor(artifactSha256).orElse(null);
+    if (reference == null) {
+      return usage(
+          "no committed reference matches this artifact"
+              + System.lineSeparator()
+              + "  artifact "
+              + artifactSha256
+              + System.lineSeparator()
+              + "  generate one, see embedding-equivalence/README.md");
+    }
     String probeSetSha256 = EmbeddingEquivalence.sha256(EmbeddingEquivalence.loadProbesRaw());
     if (!probeSetSha256.equals(reference.probeSetSha256())) {
       return usage(
           "probe set has changed since the reference was generated; regenerate it (see"
               + " embedding-equivalence/README.md)");
     }
-    String artifactSha256 = Hashing.sha256(artifact);
-    if (!artifactSha256.equals(reference.artifactSha256())) {
-      return usage(
-          "artifact does not match the one the reference was generated from"
-              + System.lineSeparator()
-              + "  expected "
-              + reference.artifactSha256()
-              + System.lineSeparator()
-              + "  actual   "
-              + artifactSha256);
-    }
 
     List<float[]> measured = new ArrayList<>(probes.size());
     List<Double> millis = new ArrayList<>(probes.size());