From 8d579966d2acc5fbb6dd2b87f2588388d2d972ea Mon Sep 17 00:00:00 2001 From: Mithun Kumar Date: Thu, 11 Dec 2025 15:31:33 +0530 Subject: [PATCH 01/21] Added Endee VectorDB --- vectordb_bench/__init__.py | 2 +- vectordb_bench/backend/clients/__init__.py | 11 + .../backend/clients/endee/__init__.py | 0 vectordb_bench/backend/clients/endee/cli.py | 114 ++ .../backend/clients/endee/config.py | 32 + vectordb_bench/backend/clients/endee/endee.py | 206 ++++ .../backend/runner/serial_runner.py | 1048 ++++++++++++++++- vectordb_bench/cli/vectordbbench.py | 2 + vectordb_bench/frontend/config/styles.py | 1 + 9 files changed, 1378 insertions(+), 38 deletions(-) create mode 100644 vectordb_bench/backend/clients/endee/__init__.py create mode 100644 vectordb_bench/backend/clients/endee/cli.py create mode 100644 vectordb_bench/backend/clients/endee/config.py create mode 100644 vectordb_bench/backend/clients/endee/endee.py diff --git a/vectordb_bench/__init__.py b/vectordb_bench/__init__.py index 07f77bb02..0a2512f7c 100644 --- a/vectordb_bench/__init__.py +++ b/vectordb_bench/__init__.py @@ -19,7 +19,7 @@ class config: DEFAULT_DATASET_URL = env.str("DEFAULT_DATASET_URL", AWS_S3_URL) DATASET_SOURCE = env.str("DATASET_SOURCE", "S3") # Options "S3" or "AliyunOSS" DATASET_LOCAL_DIR = env.path("DATASET_LOCAL_DIR", "/tmp/vectordb_bench/dataset") - NUM_PER_BATCH = env.int("NUM_PER_BATCH", 100) + NUM_PER_BATCH = env.int("NUM_PER_BATCH", 1000) TIME_PER_BATCH = 1 # 1s. for streaming insertion. MAX_INSERT_RETRY = 5 MAX_SEARCH_RETRY = 5 diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index 3bd2d3189..748f64707 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -56,6 +56,7 @@ class DB(Enum): AliSQL = "AlibabaCloudRDSMySQL" Doris = "Doris" TurboPuffer = "TurboPuffer" + Endee = "Endee" @property def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 @@ -227,6 +228,11 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 from .alisql.alisql import AliSQL return AliSQL + + if self == DB.Endee: + from .endee.endee import Endee + + return Endee msg = f"Unknown DB: {self.name}" raise ValueError(msg) @@ -401,6 +407,11 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901, PLR0915 from .alisql.config import AliSQLConfig return AliSQLConfig + + if self == DB.Endee: + from .endee.config import EndeeConfig + + return EndeeConfig msg = f"Unknown DB: {self.name}" raise ValueError(msg) diff --git a/vectordb_bench/backend/clients/endee/__init__.py b/vectordb_bench/backend/clients/endee/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vectordb_bench/backend/clients/endee/cli.py b/vectordb_bench/backend/clients/endee/cli.py new file mode 100644 index 000000000..952f1b682 --- /dev/null +++ b/vectordb_bench/backend/clients/endee/cli.py @@ -0,0 +1,114 @@ +import click +import logging +import uuid +from typing import Annotated + +from vectordb_bench.cli.cli import ( + CommonTypedDict, + benchmark_runner, + click_parameter_decorators_from_typed_dict, + get_custom_case_config, + parse_task_stages, +) +from .. import DB +from .config import EndeeConfig +from ..api import EmptyDBCaseConfig + +log = logging.getLogger(__name__) + + +class EndeeTypedDict(CommonTypedDict): + token: Annotated[ + str, + click.option("--token", type=str, required=True, help="Endee API token") + ] + region: Annotated[ + str, + click.option("--region", type=str, default=None, help="Endee region", show_default=True) + ] + space_type: Annotated[ + str, + click.option("--space-type", type=click.Choice(["cosine", "l2", "dot_product"]), default="cosine", help="Distance metric", show_default=True) + ] + precision: Annotated[ + str, + click.option("--precision", type=click.Choice(["medium", "high", "ultra-high", "fp16"]), default="medium", help="Quant Level", show_default=True) + ] + version: Annotated[ + int, + click.option("--version", type=int, default=None, help="Index version", show_default=True) + ] + m: Annotated[ + int, + click.option("--m", type=int, default=None, help="HNSW M parameter", show_default=True) + ] + ef_con: Annotated[ + int, + click.option("--ef-con", type=int, default=None, help="HNSW construction parameter", show_default=True) + ] + ef_search: Annotated[ + int, + click.option("--ef-search", type=int, default=None, help="HNSW search parameter", show_default=True) + ] + index_name: Annotated[ + str, + click.option("--index-name", type=str, required=True, help="Endee index name (will use a random name if not provided)", show_default=True) + ] + + + +@click.command() +@click_parameter_decorators_from_typed_dict(EndeeTypedDict) +def Endee(**parameters): + stages = parse_task_stages( + parameters["drop_old"], + parameters["load"], + parameters["search_serial"], + parameters["search_concurrent"], + ) + + # Generate a random collection name if not provided + collection_name = parameters["index_name"] + if not collection_name: + collection_name = f"endee_bench_{uuid.uuid4().hex[:8]}" + + # Filter out None values before creating config + params_for_vecx = {k: v for k, v in parameters.items() if v is not None} + db_config = EndeeConfig(**params_for_vecx) + + custom_case_config = get_custom_case_config(parameters) + + # Create task config + from vectordb_bench.models import TaskConfig, CaseConfig, CaseType, ConcurrencySearchConfig + + # Create an instance of EmptyDBCaseConfig instead of passing None + db_case_config = EmptyDBCaseConfig() + + task = TaskConfig( + db=DB.Endee, + db_config=db_config, # Use the EndeeConfig instance directly + db_case_config=db_case_config, + case_config=CaseConfig( + case_id=CaseType[parameters["case_type"]], + k=parameters["k"], + concurrency_search_config=ConcurrencySearchConfig( + concurrency_duration=parameters["concurrency_duration"], + num_concurrency=[int(s) for s in parameters["num_concurrency"]], + ), + custom_case=custom_case_config, + ), + stages=stages, + ) + + # Use the run method of the benchmark_runner object + if not parameters["dry_run"]: + benchmark_runner.run([task]) + + # Wait for task to complete if needed + import time + from vectordb_bench.interface import global_result_future + from concurrent.futures import wait + + time.sleep(5) + if global_result_future: + wait([global_result_future]) \ No newline at end of file diff --git a/vectordb_bench/backend/clients/endee/config.py b/vectordb_bench/backend/clients/endee/config.py new file mode 100644 index 000000000..476f9b5e0 --- /dev/null +++ b/vectordb_bench/backend/clients/endee/config.py @@ -0,0 +1,32 @@ +from pydantic import SecretStr +from typing import Optional +from ..api import DBConfig + + +class EndeeConfig(DBConfig): + token: SecretStr + region: Optional[str] = "india-west-1" + space_type: str ="cosine" + # use_fp16: bool = False + precision: str = "medium" + version: Optional[int] = 1 + m: Optional[int] = 16 + ef_con: Optional[int] = 128 + ef_search: Optional[int] = 128 + # collection_name: str + index_name: str + + def to_dict(self) -> dict: + return { + "token": self.token.get_secret_value(), + "region": self.region, + "space_type": self.space_type, + # "use_fp16": self.use_fp16, + "precision": self.precision, + "version": self.version, + "m": self.m, + "ef_con": self.ef_con, + "ef_search": self.ef_search, + # "collection_name": self.collection_name, + "index_name": self.index_name, + } \ No newline at end of file diff --git a/vectordb_bench/backend/clients/endee/endee.py b/vectordb_bench/backend/clients/endee/endee.py new file mode 100644 index 000000000..5095acd13 --- /dev/null +++ b/vectordb_bench/backend/clients/endee/endee.py @@ -0,0 +1,206 @@ +import logging +from contextlib import contextmanager + +from endee import endee + +from ..api import DBCaseConfig, EmptyDBCaseConfig, IndexType, VectorDB +from .config import EndeeConfig + +log = logging.getLogger(__name__) + + +class Endee(VectorDB): + def __init__( + self, + dim: int, + db_config: dict, + db_case_config: DBCaseConfig, + drop_old: bool = False, + **kwargs, + ): + print(db_config) + + self.token = db_config.get("token", "") + self.region = db_config.get("region", "india-west-1") + + # self.collection_name = db_config.get("collection_name", "") + self.collection_name = (db_config.get("collection_name") or db_config.get("index_name")) + + if not self.collection_name: + import uuid + self.collection_name = f"endee_bench_{uuid.uuid4().hex[:8]}" + + self.space_type = db_config.get("space_type", "cosine") + # self.use_fp16 = db_config.get("use_fp16", False) + self.precision = db_config.get("precision") + self.version = db_config.get("version") + self.M = db_config.get("m") + self.ef_con = db_config.get("ef_con") + self.ef_search = db_config.get("ef_search") + self.nd = endee.Endee(token=self.token) + + # Using Base url + # self.nd.base_url = "https://10.160.0.8:8080" + + + # try: + # indices = self.nd.list_indexes().get("indices", []) + # index_names = [index["name"] for index in indices] if indices else [] + + # if drop_old and self.collection_name in index_names: + # self._drop_index(self.collection_name) + # self._create_index(dim) + # elif self.collection_name not in index_names: + # self._create_index(dim) + # except Exception as e: + # log.error(f"Error connecting to Endee API: {e}") + # raise + + try: + index_name = self.collection_name # assign before use + indices = self.nd.list_indexes().get("indices", []) + index_names = [index["name"] for index in indices] if indices else [] + try: + self.index = self.nd.get_index(name=index_name) + log.info(f"Connected to existing Endee index: '{index_name}'") + + except Exception as fetch_error: + log.warning(f"Index '{index_name}' not found. Creating new index...") + try: + self._create_index(dim) + self.index = self.nd.get_index(name=index_name) + log.info(f"Successfully created and connected to index: '{index_name}'") + + except Exception as create_error: + # If error is "index already exists", just get the index again + if "already exists" in str(create_error).lower() or "conflict" in str(create_error).lower(): + log.warning(f"Index '{index_name}' already exists despite previous error. Fetching it again.") + self.index = self.nd.get_index(name=index_name) + else: + log.error(f"Failed to create Endee index: {create_error}") + raise + except Exception as e: + log.error(f"Error accessing or creating Endee index '{index_name}': {e}") + raise + + + + def _create_index(self, dim): + try: + resp = self.nd.create_index( + name=self.collection_name, + dimension=dim, + space_type=self.space_type, + # use_fp16=self.use_fp16, + precision=self.precision, + version=self.version, + M=self.M, + ef_con=self.ef_con + ) + log.info(f"Created new Endee index: {resp}") + except Exception as e: + log.error(f"Failed to create Endee index: {e}") + raise + + def _drop_index(self, collection_name): + try: + res = self.nd.delete_index(collection_name) + log.info(res) + except Exception as e: + log.error(f"Failed to drop Endee index: {e}") + raise + + @classmethod + def config_cls(cls) -> type[EndeeConfig]: + return EndeeConfig + + @classmethod + def case_config_cls(cls, index_type: IndexType | None = None) -> type[DBCaseConfig]: + return EmptyDBCaseConfig + + @contextmanager + def init(self): + try: + # log.info(f"Token: {self.token}") + nd = endee.Endee(token=self.token) + # Uncomment below to use base_url + # nd.base_url = "https://10.160.0.8:8080" + # self.nd = nd + self.index = nd.get_index(name=self.collection_name) + yield + except Exception as e: + if hasattr(e, 'response') and e.response is not None: + log.error(f"HTTP Status: {e.response.status_code}, Body: {e.response.text}") + log.error(f"Error initializing Endee client: {e}") + # log.error("Error initializing Endee client", exc_info=True) + raise + finally: + pass + + def optimize(self, data_size: int | None = None): + pass + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs, + ) -> (int, Exception): # type: ignore + assert len(embeddings) == len(metadata) + insert_count = 0 + try: + batch_vectors = [ + { + "id": str(metadata[i]), + "vector": embeddings[i], + "meta": {"id": str(metadata[i])} + } + for i in range(len(embeddings)) + ] + self.index.upsert(batch_vectors) + insert_count = len(batch_vectors) + + except Exception as e: + return (insert_count, e) + + return (len(embeddings), None) + + def search_embedding( + self, + query: list[float], + k: int = 30, + filters: dict | None = None, + **kwargs, + ) -> list[int]: + try: + filter_expr = None + if filters and "id" in filters: + filter_expr = {"id": filters["id"]} + + results = self.index.query( + vector=query, + top_k=k, + filter=filter_expr, + ef=self.ef_search, + include_vectors=False + ) + + return [int(result["id"]) for result in results] + + except Exception as e: + log.warning(f"Error querying Endee index: {e}") + raise + + def describe_index(self) -> dict: + """Get information about the current index.""" + try: + all_indices = self.nd.list_indexes().get("indices", []) + + for idx in all_indices: + if idx.get("name") == self.collection_name: + return idx + + return {} + except Exception as e: + log.warning(f"Error describing Endee index: {e}") + return {} diff --git a/vectordb_bench/backend/runner/serial_runner.py b/vectordb_bench/backend/runner/serial_runner.py index 300553a4e..eeb2e988b 100644 --- a/vectordb_bench/backend/runner/serial_runner.py +++ b/vectordb_bench/backend/runner/serial_runner.py @@ -17,6 +17,42 @@ from .. import utils from ..clients import api +import os +import json + +# CHECKPOINT_PATH = "/mnt/data/VectorDBBench/insert_checkpoint.json" + +# Get the path of the current script +CURRENT_FILE = os.path.abspath(__file__) + +# Go up 3 levels to reach the root benchmarking folder +ROOT_BENCHMARK_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(CURRENT_FILE)))) + +CHECKPOINT_FILENAME = "insert_checkpoint.json" +CHECKPOINT_PATH = os.path.join(ROOT_BENCHMARK_DIR, CHECKPOINT_FILENAME) + +# Ensure the root folder exists (should already exist, but safe) +os.makedirs(ROOT_BENCHMARK_DIR, exist_ok=True) + +def save_checkpoint(count: int): + with open(CHECKPOINT_PATH, "w") as f: + json.dump({"last_index": count}, f) + +def load_checkpoint() -> int: + if os.path.exists(CHECKPOINT_PATH): + try: + with open(CHECKPOINT_PATH, "r") as f: + data = json.load(f) + return data.get("last_index", 0) + except Exception as e: + log.warning(f"Failed to read checkpoint file: {e}") + return 0 + +def clear_checkpoint(): + if os.path.exists(CHECKPOINT_PATH): + os.remove(CHECKPOINT_PATH) + + NUM_PER_BATCH = config.NUM_PER_BATCH LOAD_MAX_TRY_COUNT = config.LOAD_MAX_TRY_COUNT @@ -52,20 +88,29 @@ def retry_insert(self, db: api.VectorDB, retry_idx: int = 0, **kwargs): def task(self) -> int: count = 0 + last_index = load_checkpoint() + log.info(f"Resuming insertion from checkpoint at index {last_index}") + with self.db.init(): log.info(f"({mp.current_process().name:16}) Start inserting embeddings in batch {config.NUM_PER_BATCH}") start = time.perf_counter() + for data_df in self.dataset: - all_metadata = data_df[self.dataset.data.train_id_field].tolist() + batch_size = len(data_df) + if count + batch_size <= last_index: + count += batch_size + log.debug(f"Skipping batch of size {batch_size}, total skipped={count}") + continue + all_metadata = data_df[self.dataset.data.train_id_field].tolist() emb_np = np.stack(data_df[self.dataset.data.train_vector_field]) + if self.normalize: - log.debug("normalize the 100k train data") + log.debug("Normalizing train data") all_embeddings = (emb_np / np.linalg.norm(emb_np, axis=1)[:, np.newaxis]).tolist() else: all_embeddings = emb_np.tolist() del emb_np - log.debug(f"batch dataset size: {len(all_embeddings)}, {len(all_metadata)}") labels_data = None if self.filters.type == FilterOp.StrEqual: @@ -79,6 +124,7 @@ def task(self) -> int: metadata=all_metadata, labels_data=labels_data, ) + if error is not None: self.retry_insert( self.db, @@ -89,15 +135,19 @@ def task(self) -> int: assert insert_count == len(all_metadata) count += insert_count + save_checkpoint(count) # Save progress after each batch + if count % 100_000 == 0: log.info(f"({mp.current_process().name:16}) Loaded {count} embeddings into VectorDB") log.info( - f"({mp.current_process().name:16}) Finish loading all dataset into VectorDB, " + f"({mp.current_process().name:16}) Finished loading all dataset into VectorDB, " f"dur={time.perf_counter() - start}" ) + clear_checkpoint() # Clean checkpoint file after successful completion return count + def endless_insert_data(self, all_embeddings: list, all_metadata: list, left_id: int = 0) -> int: with self.db.init(): # unique id for endlessness insertion @@ -209,7 +259,80 @@ def run(self) -> int: return count +# ============================================================================= +# PARALLEL SEARCH WORKER FUNCTION +# ============================================================================= +def _parallel_search_worker(args: tuple) -> list[tuple[float, int, list[int]]]: + """ + Worker function executed in separate process for parallel search. + + Each worker: + 1. Receives a batch of queries (embeddings) with their indices + 2. Initializes its own DB connection + 3. Executes searches for all queries in the batch + 4. Returns results with latencies and original indices + + Args: + args: Tuple of (db_instance, batch_queries, batch_indices, k, filters) + + Returns: + List of tuples: [(latency, original_query_idx, search_results), ...] + + This function runs in a subprocess, so it needs to: + - Initialize DB connection independently + - Handle its own retry logic + - Return all results for aggregation in main process + """ + db_instance, batch_queries, batch_indices, k, filters = args + + results_batch = [] + + # Each worker initializes its own DB connection + with db_instance.init(): + # Prepare filters if needed (e.g., for filtered search scenarios) + db_instance.prepare_filter(filters) + + # Process each query in this worker's batch + for idx, emb in zip(batch_indices, batch_queries): + retry_count = 0 + max_retries = config.MAX_SEARCH_RETRY + + # Retry logic for each individual query + while retry_count <= max_retries: + try: + # Time the search operation + s = time.perf_counter() + results = db_instance.search_embedding(emb, k) + latency = time.perf_counter() - s + + # Store result with original index for proper ordering later + results_batch.append((latency, idx, results)) + break # Success, move to next query + + except Exception as e: + retry_count += 1 + if retry_count > max_retries: + log.error(f"Query {idx} failed after {max_retries} retries: {e}") + raise + log.warning(f"Query {idx} retry {retry_count}/{max_retries}: {e}") + time.sleep(0.1 * retry_count) # Exponential backoff + + return results_batch + + class SerialSearchRunner: + """ + Search runner that executes queries to measure recall and latency. + + PERFORMANCE ENHANCEMENT: + This class now uses parallel processing under the hood while maintaining + the same interface. Queries are distributed across multiple worker processes + for significant speedup (4-8x faster depending on CPU cores). + + The parallelization is transparent to callers - all existing code continues + to work without modification. + """ + def __init__( self, db: api.VectorDB, @@ -222,13 +345,25 @@ def __init__( self.k = k self.filters = filters + # Convert test data to list format if needed if isinstance(test_data[0], np.ndarray): self.test_data = [query.tolist() for query in test_data] else: self.test_data = test_data self.ground_truth = ground_truth + + # PARALLEL PROCESSING CONFIG: + # Auto-detect number of CPU cores, reserve 1 for main process + # This allows efficient parallel query execution + self.num_workers = max(1, mp.cpu_count() - 1) + log.info(f"SerialSearchRunner initialized with {self.num_workers} parallel workers") def _get_db_search_res(self, emb: list[float], retry_idx: int = 0) -> list[int]: + """ + LEGACY METHOD - Kept for backward compatibility. + This method is no longer used in the main search path but retained + in case any external code calls it directly. + """ try: results = self.db.search_embedding(emb, self.k) except Exception as e: @@ -241,47 +376,141 @@ def _get_db_search_res(self, emb: list[float], retry_idx: int = 0) -> list[int]: return results + def _create_query_batches(self) -> list[tuple[list[int], list]]: + """ + PARALLEL PROCESSING HELPER: + Splits test queries into batches for distribution to worker processes. + + Each batch contains: + - Original indices (to maintain ordering after parallel execution) + - Query embeddings for this batch + + Batch size is calculated as: total_queries / num_workers + This ensures even distribution of work across all available workers. + + Returns: + List of (indices, queries) tuples, one per worker + """ + total_queries = len(self.test_data) + batch_size = max(1, math.ceil(total_queries / self.num_workers)) + batches = [] + + for i in range(0, total_queries, batch_size): + end_idx = min(i + batch_size, total_queries) + indices = list(range(i, end_idx)) + queries = self.test_data[i:end_idx] + batches.append((indices, queries)) + + log.debug(f"Created {len(batches)} query batches, batch_size={batch_size}") + return batches + def search(self, args: tuple[list, list[list[int]]]) -> tuple[float, float, float, float]: + """ + MODIFIED: Now uses parallel processing for significant speedup. + + Main changes from original: + 1. Splits queries into batches for parallel workers + 2. Spawns worker processes to execute searches concurrently + 3. Collects and re-orders results by original index + 4. Calculates metrics (recall, NDCG, latencies) from parallel results + + The interface and return values remain identical to the original version, + ensuring backward compatibility with existing code. + + Original single-threaded approach took ~164s for 1000 queries. + Parallel approach typically achieves 4-8x speedup depending on CPU cores. + + Args: + args: Tuple of (test_data, ground_truth) - kept for interface compatibility + + Returns: + Tuple of (avg_recall, avg_ndcg, p99_latency, p95_latency) + """ log.info(f"{mp.current_process().name:14} start search the entire test_data to get recall and latency") - with self.db.init(): - self.db.prepare_filter(self.filters) - test_data, ground_truth = args - ideal_dcg = get_ideal_dcg(self.k) - - log.debug(f"test dataset size: {len(test_data)}") - log.debug(f"ground truth size: {len(ground_truth)}") - - latencies, recalls, ndcgs = [], [], [] - for idx, emb in enumerate(test_data): - s = time.perf_counter() + + test_data, ground_truth = args + + # Validate input + if not test_data: + raise RuntimeError("empty test_data") + + log.debug(f"test dataset size: {len(test_data)}") + log.debug(f"ground truth size: {len(ground_truth) if ground_truth else 0}") + + # PARALLEL EXECUTION SECTION: + # Create batches for parallel processing + batches = self._create_query_batches() + + # Prepare arguments for each worker process + # Each worker gets: (db_instance, queries, indices, k, filters) + worker_args = [ + (self.db, queries, indices, self.k, self.filters) + for indices, queries in batches + ] + + log.info(f"Launching {len(worker_args)} parallel workers for search") + + # Execute searches in parallel using ProcessPoolExecutor + # Using 'spawn' context ensures clean process initialization + all_results = [] + with concurrent.futures.ProcessPoolExecutor( + max_workers=self.num_workers, + mp_context=mp.get_context("spawn") + ) as executor: + # Submit all batches to worker pool + futures = [ + executor.submit(_parallel_search_worker, args) + for args in worker_args + ] + + # Collect results as workers complete + # as_completed() allows processing results immediately when ready + for future in concurrent.futures.as_completed(futures): try: - results = self._get_db_search_res(emb) + batch_results = future.result() + all_results.extend(batch_results) except Exception as e: - log.warning(f"VectorDB search_embedding error: {e}") - raise e from None - - latencies.append(time.perf_counter() - s) - - if ground_truth is not None: - gt = ground_truth[idx] - recalls.append(calc_recall(self.k, gt[: self.k], results)) - ndcgs.append(calc_ndcg(gt[: self.k], results, ideal_dcg)) - else: - recalls.append(0) - ndcgs.append(0) - - if len(latencies) % 100 == 0: - log.debug( - f"({mp.current_process().name:14}) search_count={len(latencies):3}, " - f"latest_latency={latencies[-1]}, latest recall={recalls[-1]}" - ) - + log.error(f"Worker process failed: {e}") + raise + + # RESULTS PROCESSING SECTION: + # Sort results by original query index to maintain correct ordering + # This is crucial because workers complete in arbitrary order + all_results.sort(key=lambda x: x[1]) # Sort by index (2nd element) + + log.info(f"Collected {len(all_results)} search results from parallel workers") + + # Calculate performance metrics + latencies, recalls, ndcgs = [], [], [] + ideal_dcg = get_ideal_dcg(self.k) + + for latency, idx, results in all_results: + latencies.append(latency) + + # Calculate recall and NDCG if ground truth is available + if ground_truth is not None: + gt = ground_truth[idx] + recalls.append(calc_recall(self.k, gt[:self.k], results)) + ndcgs.append(calc_ndcg(gt[:self.k], results, ideal_dcg)) + else: + recalls.append(0) + ndcgs.append(0) + + # Progress logging every 100 queries + if len(latencies) % 100 == 0: + log.debug( + f"({mp.current_process().name:14}) search_count={len(latencies):3}, " + f"latest_latency={latencies[-1]}, latest recall={recalls[-1]}" + ) + + # Calculate aggregate statistics avg_latency = round(np.mean(latencies), 4) avg_recall = round(np.mean(recalls), 4) avg_ndcg = round(np.mean(ndcgs), 4) cost = round(np.sum(latencies), 4) p99 = round(np.percentile(latencies, 99), 4) p95 = round(np.percentile(latencies, 95), 4) + log.info( f"{mp.current_process().name:14} search entire test_data: " f"cost={cost}s, " @@ -292,15 +521,28 @@ def search(self, args: tuple[list, list[list[int]]]) -> tuple[float, float, floa f"p99={p99}, " f"p95={p95}" ) + return (avg_recall, avg_ndcg, p99, p95) def _run_in_subprocess(self) -> tuple[float, float, float, float]: + """ + UNCHANGED: Wrapper to execute search in subprocess. + + This method is kept for interface compatibility. + The actual parallelization happens inside the search() method. + """ with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor: future = executor.submit(self.search, (self.test_data, self.ground_truth)) return future.result() @utils.time_it def run(self) -> tuple[float, float, float, float]: + """ + UNCHANGED: Main entry point for search execution. + + Interface remains identical - callers don't need any changes. + Parallel processing is handled transparently. + """ log.info(f"{mp.current_process().name:14} start serial search") if self.test_data is None: msg = "empty test_data" @@ -311,9 +553,11 @@ def run(self) -> tuple[float, float, float, float]: @utils.time_it def run_with_cost(self) -> tuple[tuple[float, float, float, float], float]: """ - Search all test data in serial. + UNCHANGED: Search with cost tracking. + Returns: - tuple[tuple[float, float, float, float], float]: (avg_recall, avg_ndcg, p99_latency, p95_latency), cost + tuple[tuple[float, float, float, float], float]: + (avg_recall, avg_ndcg, p99_latency, p95_latency), cost """ log.info(f"{mp.current_process().name:14} start serial search") if self.test_data is None: @@ -321,3 +565,733 @@ def run_with_cost(self) -> tuple[tuple[float, float, float, float], float]: raise RuntimeError(msg) return self._run_in_subprocess() + + + + + + + + +# ======================================================================= + +# MODIFIED TO HAVE CHECKPOINT FOR RESUME LOGIC + +# ======================================================================= + + + + +# import concurrent +# import logging +# import math +# import multiprocessing as mp +# import time +# import traceback + +# import numpy as np +# import psutil + +# from vectordb_bench.backend.dataset import DatasetManager +# from vectordb_bench.backend.filter import Filter, FilterOp, non_filter + +# from ... import config +# from ...metric import calc_ndcg, calc_recall, get_ideal_dcg +# from ...models import LoadTimeoutError, PerformanceTimeoutError +# from .. import utils +# from ..clients import api + +# import os +# import json + + +# # Get the path of the current script +# CURRENT_FILE = os.path.abspath(__file__) + +# # Go up 3 levels to reach the root benchmarking folder +# ROOT_BENCHMARK_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(CURRENT_FILE)))) + +# CHECKPOINT_FILENAME = "insert_checkpoint.json" +# CHECKPOINT_PATH = os.path.join(ROOT_BENCHMARK_DIR, CHECKPOINT_FILENAME) + +# # Ensure the root folder exists (should already exist, but safe) +# os.makedirs(ROOT_BENCHMARK_DIR, exist_ok=True) + +# def save_checkpoint(count: int): +# with open(CHECKPOINT_PATH, "w") as f: +# json.dump({"last_index": count}, f) + +# def load_checkpoint() -> int: +# if os.path.exists(CHECKPOINT_PATH): +# try: +# with open(CHECKPOINT_PATH, "r") as f: +# data = json.load(f) +# return data.get("last_index", 0) +# except Exception as e: +# log.warning(f"Failed to read checkpoint file: {e}") +# return 0 + +# def clear_checkpoint(): +# if os.path.exists(CHECKPOINT_PATH): +# os.remove(CHECKPOINT_PATH) + + +# NUM_PER_BATCH = config.NUM_PER_BATCH +# LOAD_MAX_TRY_COUNT = config.LOAD_MAX_TRY_COUNT + +# log = logging.getLogger(__name__) + + +# class SerialInsertRunner: +# def __init__( +# self, +# db: api.VectorDB, +# dataset: DatasetManager, +# normalize: bool, +# filters: Filter = non_filter, +# timeout: float | None = None, +# ): +# self.timeout = timeout if isinstance(timeout, int | float) else None +# self.dataset = dataset +# self.db = db +# self.normalize = normalize +# self.filters = filters + +# def retry_insert(self, db: api.VectorDB, retry_idx: int = 0, **kwargs): +# _, error = db.insert_embeddings(**kwargs) +# if error is not None: +# log.warning(f"Insert Failed, try_idx={retry_idx}, Exception: {error}") +# retry_idx += 1 +# if retry_idx <= config.MAX_INSERT_RETRY: +# time.sleep(retry_idx) +# self.retry_insert(db, retry_idx=retry_idx, **kwargs) +# else: +# msg = f"Insert failed and retried more than {config.MAX_INSERT_RETRY} times" +# raise RuntimeError(msg) from None + +# def task(self) -> int: +# count = 0 +# last_index = load_checkpoint() +# log.info(f"Resuming insertion from checkpoint at index {last_index}") + +# with self.db.init(): +# log.info(f"({mp.current_process().name:16}) Start inserting embeddings in batch {config.NUM_PER_BATCH}") +# start = time.perf_counter() + +# for data_df in self.dataset: +# batch_size = len(data_df) +# if count + batch_size <= last_index: +# count += batch_size +# log.debug(f"Skipping batch of size {batch_size}, total skipped={count}") +# continue + +# all_metadata = data_df[self.dataset.data.train_id_field].tolist() +# emb_np = np.stack(data_df[self.dataset.data.train_vector_field]) + +# if self.normalize: +# log.debug("Normalizing train data") +# all_embeddings = (emb_np / np.linalg.norm(emb_np, axis=1)[:, np.newaxis]).tolist() +# else: +# all_embeddings = emb_np.tolist() +# del emb_np + +# labels_data = None +# if self.filters.type == FilterOp.StrEqual: +# if self.dataset.data.scalar_labels_file_separated: +# labels_data = self.dataset.scalar_labels[self.filters.label_field][all_metadata].to_list() +# else: +# labels_data = data_df[self.filters.label_field].tolist() + +# insert_count, error = self.db.insert_embeddings( +# embeddings=all_embeddings, +# metadata=all_metadata, +# labels_data=labels_data, +# ) + +# if error is not None: +# self.retry_insert( +# self.db, +# embeddings=all_embeddings, +# metadata=all_metadata, +# labels_data=labels_data, +# ) + +# assert insert_count == len(all_metadata) +# count += insert_count +# save_checkpoint(count) # Save progress after each batch + +# if count % 100_000 == 0: +# log.info(f"({mp.current_process().name:16}) Loaded {count} embeddings into VectorDB") + +# log.info( +# f"({mp.current_process().name:16}) Finished loading all dataset into VectorDB, " +# f"dur={time.perf_counter() - start}" +# ) +# clear_checkpoint() # Clean checkpoint file after successful completion +# return count + + +# def endless_insert_data(self, all_embeddings: list, all_metadata: list, left_id: int = 0) -> int: +# with self.db.init(): +# # unique id for endlessness insertion +# all_metadata = [i + left_id for i in all_metadata] + +# num_batches = math.ceil(len(all_embeddings) / NUM_PER_BATCH) +# log.info( +# f"({mp.current_process().name:16}) Start inserting {len(all_embeddings)} " +# f"embeddings in batch {NUM_PER_BATCH}" +# ) +# count = 0 +# for batch_id in range(num_batches): +# retry_count = 0 +# already_insert_count = 0 +# metadata = all_metadata[batch_id * NUM_PER_BATCH : (batch_id + 1) * NUM_PER_BATCH] +# embeddings = all_embeddings[batch_id * NUM_PER_BATCH : (batch_id + 1) * NUM_PER_BATCH] + +# log.debug( +# f"({mp.current_process().name:16}) batch [{batch_id:3}/{num_batches}], " +# f"Start inserting {len(metadata)} embeddings" +# ) +# while retry_count < LOAD_MAX_TRY_COUNT: +# insert_count, error = self.db.insert_embeddings( +# embeddings=embeddings[already_insert_count:], +# metadata=metadata[already_insert_count:], +# ) +# already_insert_count += insert_count +# if error is not None: +# retry_count += 1 +# time.sleep(10) + +# log.info(f"Failed to insert data, try {retry_count} time") +# if retry_count >= LOAD_MAX_TRY_COUNT: +# raise error +# else: +# break +# log.debug( +# f"({mp.current_process().name:16}) batch [{batch_id:3}/{num_batches}], " +# f"Finish inserting {len(metadata)} embeddings" +# ) + +# assert already_insert_count == len(metadata) +# count += already_insert_count +# log.info( +# f"({mp.current_process().name:16}) Finish inserting {len(all_embeddings)} embeddings in " +# f"batch {NUM_PER_BATCH}" +# ) +# return count + +# @utils.time_it +# def _insert_all_batches(self) -> int: +# """Performance case only""" +# with concurrent.futures.ProcessPoolExecutor( +# mp_context=mp.get_context("spawn"), +# max_workers=1, +# ) as executor: +# future = executor.submit(self.task) +# try: +# count = future.result(timeout=self.timeout) +# except TimeoutError as e: +# msg = f"VectorDB load dataset timeout in {self.timeout}" +# log.warning(msg) +# for pid, _ in executor._processes.items(): +# psutil.Process(pid).kill() +# raise PerformanceTimeoutError(msg) from e +# except Exception as e: +# log.warning(f"VectorDB load dataset error: {e}") +# raise e from e +# else: +# return count + +# def run_endlessness(self) -> int: +# """run forever util DB raises exception or crash""" +# # datasets for load tests are quite small, can fit into memory +# # only 1 file +# data_df = next(iter(self.dataset)) +# all_embeddings, all_metadata = ( +# np.stack(data_df[self.dataset.data.train_vector_field]).tolist(), +# data_df[self.dataset.data.train_id_field].tolist(), +# ) + +# start_time = time.perf_counter() +# max_load_count, times = 0, 0 +# try: +# while time.perf_counter() - start_time < self.timeout: +# count = self.endless_insert_data( +# all_embeddings, +# all_metadata, +# left_id=max_load_count, +# ) +# max_load_count += count +# times += 1 +# log.info( +# f"Loaded {times} entire dataset, current max load counts={utils.numerize(max_load_count)}, " +# f"{max_load_count}" +# ) +# except Exception as e: +# log.info( +# f"Capacity case load reach limit, insertion counts={utils.numerize(max_load_count)}, " +# f"{max_load_count}, err={e}" +# ) +# traceback.print_exc() +# return max_load_count +# else: +# raise LoadTimeoutError(self.timeout) + +# def run(self) -> int: +# count, _ = self._insert_all_batches() +# return count + + +# class SerialSearchRunner: +# def __init__( +# self, +# db: api.VectorDB, +# test_data: list[list[float]], +# ground_truth: list[list[int]], +# k: int = 100, +# filters: Filter = non_filter, +# ): +# self.db = db +# self.k = k +# self.filters = filters + +# if isinstance(test_data[0], np.ndarray): +# self.test_data = [query.tolist() for query in test_data] +# else: +# self.test_data = test_data +# self.ground_truth = ground_truth + +# def _get_db_search_res(self, emb: list[float], retry_idx: int = 0) -> list[int]: +# try: +# results = self.db.search_embedding(emb, self.k) +# except Exception as e: +# log.warning(f"Serial search failed, retry_idx={retry_idx}, Exception: {e}") +# if retry_idx < config.MAX_SEARCH_RETRY: +# return self._get_db_search_res(emb=emb, retry_idx=retry_idx + 1) + +# msg = f"Serial search failed and retried more than {config.MAX_SEARCH_RETRY} times" +# raise RuntimeError(msg) from e + +# return results + +# def search(self, args: tuple[list, list[list[int]]]) -> tuple[float, float, float, float]: +# log.info(f"{mp.current_process().name:14} start search the entire test_data to get recall and latency") +# with self.db.init(): +# self.db.prepare_filter(self.filters) +# test_data, ground_truth = args +# ideal_dcg = get_ideal_dcg(self.k) + +# log.debug(f"test dataset size: {len(test_data)}") +# log.debug(f"ground truth size: {len(ground_truth)}") + +# latencies, recalls, ndcgs = [], [], [] +# for idx, emb in enumerate(test_data): +# s = time.perf_counter() +# try: +# results = self._get_db_search_res(emb) +# except Exception as e: +# log.warning(f"VectorDB search_embedding error: {e}") +# raise e from None + +# latencies.append(time.perf_counter() - s) + +# if ground_truth is not None: +# gt = ground_truth[idx] +# recalls.append(calc_recall(self.k, gt[: self.k], results)) +# ndcgs.append(calc_ndcg(gt[: self.k], results, ideal_dcg)) +# else: +# recalls.append(0) +# ndcgs.append(0) + +# if len(latencies) % 100 == 0: +# log.debug( +# f"({mp.current_process().name:14}) search_count={len(latencies):3}, " +# f"latest_latency={latencies[-1]}, latest recall={recalls[-1]}" +# ) + +# avg_latency = round(np.mean(latencies), 4) +# avg_recall = round(np.mean(recalls), 4) +# avg_ndcg = round(np.mean(ndcgs), 4) +# cost = round(np.sum(latencies), 4) +# p99 = round(np.percentile(latencies, 99), 4) +# p95 = round(np.percentile(latencies, 95), 4) +# log.info( +# f"{mp.current_process().name:14} search entire test_data: " +# f"cost={cost}s, " +# f"queries={len(latencies)}, " +# f"avg_recall={avg_recall}, " +# f"avg_ndcg={avg_ndcg}, " +# f"avg_latency={avg_latency}, " +# f"p99={p99}, " +# f"p95={p95}" +# ) +# return (avg_recall, avg_ndcg, p99, p95) + +# def _run_in_subprocess(self) -> tuple[float, float, float, float]: +# with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor: +# future = executor.submit(self.search, (self.test_data, self.ground_truth)) +# return future.result() + +# @utils.time_it +# def run(self) -> tuple[float, float, float, float]: +# log.info(f"{mp.current_process().name:14} start serial search") +# if self.test_data is None: +# msg = "empty test_data" +# raise RuntimeError(msg) + +# return self._run_in_subprocess() + +# @utils.time_it +# def run_with_cost(self) -> tuple[tuple[float, float, float, float], float]: +# """ +# Search all test data in serial. +# Returns: +# tuple[tuple[float, float, float, float], float]: (avg_recall, avg_ndcg, p99_latency, p95_latency), cost +# """ +# log.info(f"{mp.current_process().name:14} start serial search") +# if self.test_data is None: +# msg = "empty test_data" +# raise RuntimeError(msg) + +# return self._run_in_subprocess() + + + + + + +# ======================================================================= + +# ORIGINAL SERIAL RUNNER CODE + +# ======================================================================= + + + + + + + +# import concurrent +# import logging +# import math +# import multiprocessing as mp +# import time +# import traceback + +# import numpy as np +# import psutil + +# from vectordb_bench.backend.dataset import DatasetManager +# from vectordb_bench.backend.filter import Filter, FilterOp, non_filter + +# from ... import config +# from ...metric import calc_ndcg, calc_recall, get_ideal_dcg +# from ...models import LoadTimeoutError, PerformanceTimeoutError +# from .. import utils +# from ..clients import api + +# NUM_PER_BATCH = config.NUM_PER_BATCH +# LOAD_MAX_TRY_COUNT = config.LOAD_MAX_TRY_COUNT + +# log = logging.getLogger(__name__) + + +# class SerialInsertRunner: +# def __init__( +# self, +# db: api.VectorDB, +# dataset: DatasetManager, +# normalize: bool, +# filters: Filter = non_filter, +# timeout: float | None = None, +# ): +# self.timeout = timeout if isinstance(timeout, int | float) else None +# self.dataset = dataset +# self.db = db +# self.normalize = normalize +# self.filters = filters + +# def retry_insert(self, db: api.VectorDB, retry_idx: int = 0, **kwargs): +# _, error = db.insert_embeddings(**kwargs) +# if error is not None: +# log.warning(f"Insert Failed, try_idx={retry_idx}, Exception: {error}") +# retry_idx += 1 +# if retry_idx <= config.MAX_INSERT_RETRY: +# time.sleep(retry_idx) +# self.retry_insert(db, retry_idx=retry_idx, **kwargs) +# else: +# msg = f"Insert failed and retried more than {config.MAX_INSERT_RETRY} times" +# raise RuntimeError(msg) from None + +# def task(self) -> int: +# count = 0 +# with self.db.init(): +# log.info(f"({mp.current_process().name:16}) Start inserting embeddings in batch {config.NUM_PER_BATCH}") +# start = time.perf_counter() +# for data_df in self.dataset: +# all_metadata = data_df[self.dataset.data.train_id_field].tolist() + +# emb_np = np.stack(data_df[self.dataset.data.train_vector_field]) +# if self.normalize: +# log.debug("normalize the 100k train data") +# all_embeddings = (emb_np / np.linalg.norm(emb_np, axis=1)[:, np.newaxis]).tolist() +# else: +# all_embeddings = emb_np.tolist() +# del emb_np +# log.debug(f"batch dataset size: {len(all_embeddings)}, {len(all_metadata)}") + +# labels_data = None +# if self.filters.type == FilterOp.StrEqual: +# if self.dataset.data.scalar_labels_file_separated: +# labels_data = self.dataset.scalar_labels[self.filters.label_field][all_metadata].to_list() +# else: +# labels_data = data_df[self.filters.label_field].tolist() + +# insert_count, error = self.db.insert_embeddings( +# embeddings=all_embeddings, +# metadata=all_metadata, +# labels_data=labels_data, +# ) +# if error is not None: +# self.retry_insert( +# self.db, +# embeddings=all_embeddings, +# metadata=all_metadata, +# labels_data=labels_data, +# ) + +# assert insert_count == len(all_metadata) +# count += insert_count +# if count % 100_000 == 0: +# log.info(f"({mp.current_process().name:16}) Loaded {count} embeddings into VectorDB") + +# log.info( +# f"({mp.current_process().name:16}) Finish loading all dataset into VectorDB, " +# f"dur={time.perf_counter() - start}" +# ) +# return count + +# def endless_insert_data(self, all_embeddings: list, all_metadata: list, left_id: int = 0) -> int: +# with self.db.init(): +# # unique id for endlessness insertion +# all_metadata = [i + left_id for i in all_metadata] + +# num_batches = math.ceil(len(all_embeddings) / NUM_PER_BATCH) +# log.info( +# f"({mp.current_process().name:16}) Start inserting {len(all_embeddings)} " +# f"embeddings in batch {NUM_PER_BATCH}" +# ) +# count = 0 +# for batch_id in range(num_batches): +# retry_count = 0 +# already_insert_count = 0 +# metadata = all_metadata[batch_id * NUM_PER_BATCH : (batch_id + 1) * NUM_PER_BATCH] +# embeddings = all_embeddings[batch_id * NUM_PER_BATCH : (batch_id + 1) * NUM_PER_BATCH] + +# log.debug( +# f"({mp.current_process().name:16}) batch [{batch_id:3}/{num_batches}], " +# f"Start inserting {len(metadata)} embeddings" +# ) +# while retry_count < LOAD_MAX_TRY_COUNT: +# insert_count, error = self.db.insert_embeddings( +# embeddings=embeddings[already_insert_count:], +# metadata=metadata[already_insert_count:], +# ) +# already_insert_count += insert_count +# if error is not None: +# retry_count += 1 +# time.sleep(10) + +# log.info(f"Failed to insert data, try {retry_count} time") +# if retry_count >= LOAD_MAX_TRY_COUNT: +# raise error +# else: +# break +# log.debug( +# f"({mp.current_process().name:16}) batch [{batch_id:3}/{num_batches}], " +# f"Finish inserting {len(metadata)} embeddings" +# ) + +# assert already_insert_count == len(metadata) +# count += already_insert_count +# log.info( +# f"({mp.current_process().name:16}) Finish inserting {len(all_embeddings)} embeddings in " +# f"batch {NUM_PER_BATCH}" +# ) +# return count + +# @utils.time_it +# def _insert_all_batches(self) -> int: +# """Performance case only""" +# with concurrent.futures.ProcessPoolExecutor( +# mp_context=mp.get_context("spawn"), +# max_workers=1, +# ) as executor: +# future = executor.submit(self.task) +# try: +# count = future.result(timeout=self.timeout) +# except TimeoutError as e: +# msg = f"VectorDB load dataset timeout in {self.timeout}" +# log.warning(msg) +# for pid, _ in executor._processes.items(): +# psutil.Process(pid).kill() +# raise PerformanceTimeoutError(msg) from e +# except Exception as e: +# log.warning(f"VectorDB load dataset error: {e}") +# raise e from e +# else: +# return count + +# def run_endlessness(self) -> int: +# """run forever util DB raises exception or crash""" +# # datasets for load tests are quite small, can fit into memory +# # only 1 file +# data_df = next(iter(self.dataset)) +# all_embeddings, all_metadata = ( +# np.stack(data_df[self.dataset.data.train_vector_field]).tolist(), +# data_df[self.dataset.data.train_id_field].tolist(), +# ) + +# start_time = time.perf_counter() +# max_load_count, times = 0, 0 +# try: +# while time.perf_counter() - start_time < self.timeout: +# count = self.endless_insert_data( +# all_embeddings, +# all_metadata, +# left_id=max_load_count, +# ) +# max_load_count += count +# times += 1 +# log.info( +# f"Loaded {times} entire dataset, current max load counts={utils.numerize(max_load_count)}, " +# f"{max_load_count}" +# ) +# except Exception as e: +# log.info( +# f"Capacity case load reach limit, insertion counts={utils.numerize(max_load_count)}, " +# f"{max_load_count}, err={e}" +# ) +# traceback.print_exc() +# return max_load_count +# else: +# raise LoadTimeoutError(self.timeout) + +# def run(self) -> int: +# count, _ = self._insert_all_batches() +# return count + + +# class SerialSearchRunner: +# def __init__( +# self, +# db: api.VectorDB, +# test_data: list[list[float]], +# ground_truth: list[list[int]], +# k: int = 100, +# filters: Filter = non_filter, +# ): +# self.db = db +# self.k = k +# self.filters = filters + +# if isinstance(test_data[0], np.ndarray): +# self.test_data = [query.tolist() for query in test_data] +# else: +# self.test_data = test_data +# self.ground_truth = ground_truth + +# def _get_db_search_res(self, emb: list[float], retry_idx: int = 0) -> list[int]: +# try: +# results = self.db.search_embedding(emb, self.k) +# except Exception as e: +# log.warning(f"Serial search failed, retry_idx={retry_idx}, Exception: {e}") +# if retry_idx < config.MAX_SEARCH_RETRY: +# return self._get_db_search_res(emb=emb, retry_idx=retry_idx + 1) + +# msg = f"Serial search failed and retried more than {config.MAX_SEARCH_RETRY} times" +# raise RuntimeError(msg) from e + +# return results + +# def search(self, args: tuple[list, list[list[int]]]) -> tuple[float, float, float, float]: +# log.info(f"{mp.current_process().name:14} start search the entire test_data to get recall and latency") +# with self.db.init(): +# self.db.prepare_filter(self.filters) +# test_data, ground_truth = args +# ideal_dcg = get_ideal_dcg(self.k) + +# log.debug(f"test dataset size: {len(test_data)}") +# log.debug(f"ground truth size: {len(ground_truth)}") + +# latencies, recalls, ndcgs = [], [], [] +# for idx, emb in enumerate(test_data): +# s = time.perf_counter() +# try: +# results = self._get_db_search_res(emb) +# except Exception as e: +# log.warning(f"VectorDB search_embedding error: {e}") +# raise e from None + +# latencies.append(time.perf_counter() - s) + +# if ground_truth is not None: +# gt = ground_truth[idx] +# recalls.append(calc_recall(self.k, gt[: self.k], results)) +# ndcgs.append(calc_ndcg(gt[: self.k], results, ideal_dcg)) +# else: +# recalls.append(0) +# ndcgs.append(0) + +# if len(latencies) % 100 == 0: +# log.debug( +# f"({mp.current_process().name:14}) search_count={len(latencies):3}, " +# f"latest_latency={latencies[-1]}, latest recall={recalls[-1]}" +# ) + +# avg_latency = round(np.mean(latencies), 4) +# avg_recall = round(np.mean(recalls), 4) +# avg_ndcg = round(np.mean(ndcgs), 4) +# cost = round(np.sum(latencies), 4) +# p99 = round(np.percentile(latencies, 99), 4) +# p95 = round(np.percentile(latencies, 95), 4) +# log.info( +# f"{mp.current_process().name:14} search entire test_data: " +# f"cost={cost}s, " +# f"queries={len(latencies)}, " +# f"avg_recall={avg_recall}, " +# f"avg_ndcg={avg_ndcg}, " +# f"avg_latency={avg_latency}, " +# f"p99={p99}, " +# f"p95={p95}" +# ) +# return (avg_recall, avg_ndcg, p99, p95) + +# def _run_in_subprocess(self) -> tuple[float, float, float, float]: +# with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor: +# future = executor.submit(self.search, (self.test_data, self.ground_truth)) +# return future.result() + +# @utils.time_it +# def run(self) -> tuple[float, float, float, float]: +# log.info(f"{mp.current_process().name:14} start serial search") +# if self.test_data is None: +# msg = "empty test_data" +# raise RuntimeError(msg) + +# return self._run_in_subprocess() + +# @utils.time_it +# def run_with_cost(self) -> tuple[tuple[float, float, float, float], float]: +# """ +# Search all test data in serial. +# Returns: +# tuple[tuple[float, float, float, float], float]: (avg_recall, avg_ndcg, p99_latency, p95_latency), cost +# """ +# log.info(f"{mp.current_process().name:14} start serial search") +# if self.test_data is None: +# msg = "empty test_data" +# raise RuntimeError(msg) + +# return self._run_in_subprocess() diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index a53f0d314..6a3765ff1 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -10,6 +10,7 @@ ElasticCloudHNSWInt4, ElasticCloudHNSWInt8, ) +from ..backend.clients.endee.cli import Endee from ..backend.clients.hologres.cli import HologresHGraph from ..backend.clients.lancedb.cli import LanceDB from ..backend.clients.mariadb.cli import MariaDBHNSW @@ -70,6 +71,7 @@ cli.add_command(AliSQLHNSW) cli.add_command(Doris) cli.add_command(TurboPuffer) +cli.add_command(Endee) if __name__ == "__main__": diff --git a/vectordb_bench/frontend/config/styles.py b/vectordb_bench/frontend/config/styles.py index bce4561fd..d14ca877e 100644 --- a/vectordb_bench/frontend/config/styles.py +++ b/vectordb_bench/frontend/config/styles.py @@ -71,6 +71,7 @@ def getPatternShape(i): DB.Doris: "https://doris.apache.org/images/logo.svg", DB.TurboPuffer: "https://turbopuffer.com/logo2.png", DB.CockroachDB: "https://raw.githubusercontent.com/cockroachdb/cockroach/master/docs/media/cockroach_db.png", + DB.Endee: "data:image/svg+xml,%3c?xml%20version=%271.0%27%20encoding=%27UTF-8%27?%3e%3csvg%20id=%27Layer_1%27%20xmlns=%27http://www.w3.org/2000/svg%27%20version=%271.1%27%20viewBox=%270%200%20600%20600%27%3e%3c!--%20Generator:%20Adobe%20Illustrator%2030.0.0,%20SVG%20Export%20Plug-In%20.%20SVG%20Version:%202.1.1%20Build%20123)%20--%3e%3cdefs%3e%3cstyle%3e%20.st0%20{%20fill:%20%233266a4;%20}%20%3c/style%3e%3c/defs%3e%3cpath%20class=%27st0%27%20d=%27M106.22,490.02H10.36l-.04-184.85c11.19-163.31,232.1-211.74,306.42-61.94,23.96,48.3,15.59,99.83,17,152.01h61.62c15.25,0,42.7-13.75,54.75-23.2,66.11-51.86,57.28-158.2-16.28-198.49-9.65-5.28-33.16-14.19-43.74-14.19h-154.31v-91.09c0-.87,2.55-1.86,3.63-1.63,104.05,4.23,201.15-21.64,284.48,55.84,119.18,110.8,69.12,325.47-91.33,362.6-28.65,6.63-85.47,7.76-115.02,4.8-19.71-1.97-43.29-16.57-55.97-31.45-37.98-44.56-20.77-98.07-24.7-151.16-5.18-69.99-100-85.31-125.9-20.47-1.16,2.92-4.76,13.88-4.76,16.3v186.91Z%27/%3e%3c/svg%3e", } # RedisCloud color: #0D6EFD From ae6c4937caa9f3b24f3f2409735bb53c61434ea4 Mon Sep 17 00:00:00 2001 From: Mithun Kumar Date: Wed, 24 Dec 2025 17:56:40 +0530 Subject: [PATCH 02/21] Added base_url parameter and resume checkpoint with index name --- .allcommands | 70 ++ vectordb_bench/backend/clients/endee/cli.py | 4 + .../backend/clients/endee/config.py | 2 + vectordb_bench/backend/clients/endee/endee.py | 26 +- .../backend/runner/serial_runner.py | 939 +++++++++++++----- 5 files changed, 771 insertions(+), 270 deletions(-) create mode 100644 .allcommands diff --git a/.allcommands b/.allcommands new file mode 100644 index 000000000..355251929 --- /dev/null +++ b/.allcommands @@ -0,0 +1,70 @@ +# FOR CREATING A NEW INDEX AND START THE PROCESS +DATASET_LOCAL_DIR="/home/shaleen/vectordataset" vectordbbench endee \ + --token "as1prod:nb5l9VEBrgDEsvjut8Oawgcbf5jghoN5" \ + --region india-west-1 \ + --base-url "https://as1.endee.io/api/v1" \ + --index-name test_prod_10M_medium_3 \ + --task-label "20251224" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision medium \ + --version 1 \ + --case-type Performance768D10M \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + +PROD: "https://as1.endee.io/api/v1" +AWS: http://54.89.169.48:80/api/v1 + + DATASET_LOCAL_DIR="/home/shaleen/vectordataset" vectordbbench endee \ + --token "awsbench16:uVQMSo85De3k1lN5FiZI0xKcwXDlyBvu" \ + --region india-west-1 \ + --base-url "http://54.89.169.48:80/api/v1"\ + --index-name test_aws_10M_medium_1 \ + --task-label "20251223" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision medium \ + --version 1 \ + --case-type Performance768D10M \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + +# FOR USING THE EXISTING INDEX +DATASET_LOCAL_DIR="/home/shaleen/vectordataset" vectordbbench endee \ + --token "as1prod:nb5l9VEBrgDEsvjut8Oawgcbf5jghoN5" \ + --region india-west-1 \ + --base-url "https://as1.endee.io/api/v1" \ + --index-name test_prod_10M_medium_3 \ + --task-label "20251224" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision medium \ + --version 1 \ + --case-type Performance768D10M \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial diff --git a/vectordb_bench/backend/clients/endee/cli.py b/vectordb_bench/backend/clients/endee/cli.py index 952f1b682..e6096fc2e 100644 --- a/vectordb_bench/backend/clients/endee/cli.py +++ b/vectordb_bench/backend/clients/endee/cli.py @@ -26,6 +26,10 @@ class EndeeTypedDict(CommonTypedDict): str, click.option("--region", type=str, default=None, help="Endee region", show_default=True) ] + base_url: Annotated[ + str, + click.option("--base-url", type=str, default="http://127.0.0.1:8080/api/v1", help="API server URL", show_default=True) + ] space_type: Annotated[ str, click.option("--space-type", type=click.Choice(["cosine", "l2", "dot_product"]), default="cosine", help="Distance metric", show_default=True) diff --git a/vectordb_bench/backend/clients/endee/config.py b/vectordb_bench/backend/clients/endee/config.py index 476f9b5e0..aa4bf8c5d 100644 --- a/vectordb_bench/backend/clients/endee/config.py +++ b/vectordb_bench/backend/clients/endee/config.py @@ -6,6 +6,7 @@ class EndeeConfig(DBConfig): token: SecretStr region: Optional[str] = "india-west-1" + base_url: str = "http://127.0.0.1:8080/api/v1" # Default value space_type: str ="cosine" # use_fp16: bool = False precision: str = "medium" @@ -20,6 +21,7 @@ def to_dict(self) -> dict: return { "token": self.token.get_secret_value(), "region": self.region, + "base_url": self.base_url, "space_type": self.space_type, # "use_fp16": self.use_fp16, "precision": self.precision, diff --git a/vectordb_bench/backend/clients/endee/endee.py b/vectordb_bench/backend/clients/endee/endee.py index 5095acd13..a87efbf37 100644 --- a/vectordb_bench/backend/clients/endee/endee.py +++ b/vectordb_bench/backend/clients/endee/endee.py @@ -22,6 +22,7 @@ def __init__( self.token = db_config.get("token", "") self.region = db_config.get("region", "india-west-1") + self.base_url = db_config.get("base_url") # self.collection_name = db_config.get("collection_name", "") self.collection_name = (db_config.get("collection_name") or db_config.get("index_name")) @@ -39,9 +40,20 @@ def __init__( self.ef_search = db_config.get("ef_search") self.nd = endee.Endee(token=self.token) + # Dynamically set the URL + if self.base_url: + self.nd.set_base_url(self.base_url) + log.info(f"Targeting server: {self.base_url}") + # Using Base url - # self.nd.base_url = "https://10.160.0.8:8080" + # self.nd.base_url = "http://10.128.0.3:8080/api/v1" + # self.nd.set_base_url("http://10.128.0.3:8080/api/v1") + # self.nd.set_base_url("http://3.85.217.253:80/api/v1") + # self.nd.set_base_url("http://10.128.0.5:8080/api/v1") + # self.nd.set_base_url("http://54.89.169.48:80/api/v1") + + # BASE_URL="http://3.85.217.253:80/api/v1" # try: # indices = self.nd.list_indexes().get("indices", []) @@ -124,8 +136,16 @@ def init(self): # log.info(f"Token: {self.token}") nd = endee.Endee(token=self.token) # Uncomment below to use base_url - # nd.base_url = "https://10.160.0.8:8080" - # self.nd = nd + # nd.base_url = "http://10.128.0.3:8080/api/v1" + # nd.set_base_url("http://10.128.0.3:8080/api/v1") + # nd.set_base_url("http://3.85.217.253:80/api/v1") + # nd.set_base_url("http://10.128.0.5:8080/api/v1") + # nd.set_base_url("http://54.89.169.48:80/api/v1") + + if self.base_url: + nd.set_base_url(self.base_url) + self.nd = nd + self.index = nd.get_index(name=self.collection_name) yield except Exception as e: diff --git a/vectordb_bench/backend/runner/serial_runner.py b/vectordb_bench/backend/runner/serial_runner.py index eeb2e988b..5a4c2d0e3 100644 --- a/vectordb_bench/backend/runner/serial_runner.py +++ b/vectordb_bench/backend/runner/serial_runner.py @@ -4,7 +4,8 @@ import multiprocessing as mp import time import traceback - +import os +import json import numpy as np import psutil @@ -17,42 +18,15 @@ from .. import utils from ..clients import api -import os -import json - -# CHECKPOINT_PATH = "/mnt/data/VectorDBBench/insert_checkpoint.json" - -# Get the path of the current script +# Get the path of the current script to resolve root dir CURRENT_FILE = os.path.abspath(__file__) # Go up 3 levels to reach the root benchmarking folder ROOT_BENCHMARK_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(CURRENT_FILE)))) -CHECKPOINT_FILENAME = "insert_checkpoint.json" -CHECKPOINT_PATH = os.path.join(ROOT_BENCHMARK_DIR, CHECKPOINT_FILENAME) - # Ensure the root folder exists (should already exist, but safe) os.makedirs(ROOT_BENCHMARK_DIR, exist_ok=True) -def save_checkpoint(count: int): - with open(CHECKPOINT_PATH, "w") as f: - json.dump({"last_index": count}, f) - -def load_checkpoint() -> int: - if os.path.exists(CHECKPOINT_PATH): - try: - with open(CHECKPOINT_PATH, "r") as f: - data = json.load(f) - return data.get("last_index", 0) - except Exception as e: - log.warning(f"Failed to read checkpoint file: {e}") - return 0 - -def clear_checkpoint(): - if os.path.exists(CHECKPOINT_PATH): - os.remove(CHECKPOINT_PATH) - - NUM_PER_BATCH = config.NUM_PER_BATCH LOAD_MAX_TRY_COUNT = config.LOAD_MAX_TRY_COUNT @@ -74,6 +48,42 @@ def __init__( self.normalize = normalize self.filters = filters + # --- DYNAMIC CHECKPOINT LOGIC --- + # Generate a unique checkpoint file based on the index/collection name + # This prevents parallel processes from overwriting each other's progress. + collection_name = getattr(self.db, "collection_name", "unknown_index") + safe_name = "".join(c for c in collection_name if c.isalnum() or c in ('_', '-')) + self.checkpoint_path = os.path.join(ROOT_BENCHMARK_DIR, f"insert_checkpoint_{safe_name}.json") + log.info(f"Using unique checkpoint path: {self.checkpoint_path}") + + def _save_checkpoint(self, count: int): + """Saves current insertion progress to the dynamic JSON file.""" + try: + with open(self.checkpoint_path, "w") as f: + json.dump({"last_index": count}, f) + except Exception as e: + log.warning(f"Failed to save checkpoint: {e}") + + def _load_checkpoint(self) -> int: + """Reads progress from the dynamic JSON file if it exists.""" + if os.path.exists(self.checkpoint_path): + try: + with open(self.checkpoint_path, "r") as f: + data = json.load(f) + return data.get("last_index", 0) + except Exception as e: + log.warning(f"Failed to read checkpoint file: {e}") + return 0 + + def _clear_checkpoint(self): + """Removes the checkpoint file upon successful completion of the task.""" + if os.path.exists(self.checkpoint_path): + try: + os.remove(self.checkpoint_path) + log.info(f"Cleared checkpoint file: {self.checkpoint_path}") + except Exception as e: + log.warning(f"Failed to remove checkpoint file: {e}") + def retry_insert(self, db: api.VectorDB, retry_idx: int = 0, **kwargs): _, error = db.insert_embeddings(**kwargs) if error is not None: @@ -88,8 +98,9 @@ def retry_insert(self, db: api.VectorDB, retry_idx: int = 0, **kwargs): def task(self) -> int: count = 0 - last_index = load_checkpoint() - log.info(f"Resuming insertion from checkpoint at index {last_index}") + last_index = self._load_checkpoint() + if last_index > 0: + log.info(f"Resuming insertion from checkpoint at index {last_index}") with self.db.init(): log.info(f"({mp.current_process().name:16}) Start inserting embeddings in batch {config.NUM_PER_BATCH}") @@ -99,7 +110,9 @@ def task(self) -> int: batch_size = len(data_df) if count + batch_size <= last_index: count += batch_size - log.debug(f"Skipping batch of size {batch_size}, total skipped={count}") + # Log skip progress occasionally to avoid silence + if count % 100_000 == 0: + log.debug(f"Skipping batch of size {batch_size}, total skipped={count}") continue all_metadata = data_df[self.dataset.data.train_id_field].tolist() @@ -135,7 +148,9 @@ def task(self) -> int: assert insert_count == len(all_metadata) count += insert_count - save_checkpoint(count) # Save progress after each batch + + # Save progress after each batch to the dynamic path + self._save_checkpoint(count) if count % 100_000 == 0: log.info(f"({mp.current_process().name:16}) Loaded {count} embeddings into VectorDB") @@ -144,7 +159,9 @@ def task(self) -> int: f"({mp.current_process().name:16}) Finished loading all dataset into VectorDB, " f"dur={time.perf_counter() - start}" ) - clear_checkpoint() # Clean checkpoint file after successful completion + + # Clean checkpoint file ONLY after successful completion + self._clear_checkpoint() return count @@ -259,80 +276,7 @@ def run(self) -> int: return count -# ============================================================================= -# PARALLEL SEARCH WORKER FUNCTION -# ============================================================================= -def _parallel_search_worker(args: tuple) -> list[tuple[float, int, list[int]]]: - """ - Worker function executed in separate process for parallel search. - - Each worker: - 1. Receives a batch of queries (embeddings) with their indices - 2. Initializes its own DB connection - 3. Executes searches for all queries in the batch - 4. Returns results with latencies and original indices - - Args: - args: Tuple of (db_instance, batch_queries, batch_indices, k, filters) - - Returns: - List of tuples: [(latency, original_query_idx, search_results), ...] - - This function runs in a subprocess, so it needs to: - - Initialize DB connection independently - - Handle its own retry logic - - Return all results for aggregation in main process - """ - db_instance, batch_queries, batch_indices, k, filters = args - - results_batch = [] - - # Each worker initializes its own DB connection - with db_instance.init(): - # Prepare filters if needed (e.g., for filtered search scenarios) - db_instance.prepare_filter(filters) - - # Process each query in this worker's batch - for idx, emb in zip(batch_indices, batch_queries): - retry_count = 0 - max_retries = config.MAX_SEARCH_RETRY - - # Retry logic for each individual query - while retry_count <= max_retries: - try: - # Time the search operation - s = time.perf_counter() - results = db_instance.search_embedding(emb, k) - latency = time.perf_counter() - s - - # Store result with original index for proper ordering later - results_batch.append((latency, idx, results)) - break # Success, move to next query - - except Exception as e: - retry_count += 1 - if retry_count > max_retries: - log.error(f"Query {idx} failed after {max_retries} retries: {e}") - raise - log.warning(f"Query {idx} retry {retry_count}/{max_retries}: {e}") - time.sleep(0.1 * retry_count) # Exponential backoff - - return results_batch - - class SerialSearchRunner: - """ - Search runner that executes queries to measure recall and latency. - - PERFORMANCE ENHANCEMENT: - This class now uses parallel processing under the hood while maintaining - the same interface. Queries are distributed across multiple worker processes - for significant speedup (4-8x faster depending on CPU cores). - - The parallelization is transparent to callers - all existing code continues - to work without modification. - """ - def __init__( self, db: api.VectorDB, @@ -345,25 +289,13 @@ def __init__( self.k = k self.filters = filters - # Convert test data to list format if needed if isinstance(test_data[0], np.ndarray): self.test_data = [query.tolist() for query in test_data] else: self.test_data = test_data self.ground_truth = ground_truth - - # PARALLEL PROCESSING CONFIG: - # Auto-detect number of CPU cores, reserve 1 for main process - # This allows efficient parallel query execution - self.num_workers = max(1, mp.cpu_count() - 1) - log.info(f"SerialSearchRunner initialized with {self.num_workers} parallel workers") def _get_db_search_res(self, emb: list[float], retry_idx: int = 0) -> list[int]: - """ - LEGACY METHOD - Kept for backward compatibility. - This method is no longer used in the main search path but retained - in case any external code calls it directly. - """ try: results = self.db.search_embedding(emb, self.k) except Exception as e: @@ -376,141 +308,47 @@ def _get_db_search_res(self, emb: list[float], retry_idx: int = 0) -> list[int]: return results - def _create_query_batches(self) -> list[tuple[list[int], list]]: - """ - PARALLEL PROCESSING HELPER: - Splits test queries into batches for distribution to worker processes. - - Each batch contains: - - Original indices (to maintain ordering after parallel execution) - - Query embeddings for this batch - - Batch size is calculated as: total_queries / num_workers - This ensures even distribution of work across all available workers. - - Returns: - List of (indices, queries) tuples, one per worker - """ - total_queries = len(self.test_data) - batch_size = max(1, math.ceil(total_queries / self.num_workers)) - batches = [] - - for i in range(0, total_queries, batch_size): - end_idx = min(i + batch_size, total_queries) - indices = list(range(i, end_idx)) - queries = self.test_data[i:end_idx] - batches.append((indices, queries)) - - log.debug(f"Created {len(batches)} query batches, batch_size={batch_size}") - return batches - def search(self, args: tuple[list, list[list[int]]]) -> tuple[float, float, float, float]: - """ - MODIFIED: Now uses parallel processing for significant speedup. - - Main changes from original: - 1. Splits queries into batches for parallel workers - 2. Spawns worker processes to execute searches concurrently - 3. Collects and re-orders results by original index - 4. Calculates metrics (recall, NDCG, latencies) from parallel results - - The interface and return values remain identical to the original version, - ensuring backward compatibility with existing code. - - Original single-threaded approach took ~164s for 1000 queries. - Parallel approach typically achieves 4-8x speedup depending on CPU cores. - - Args: - args: Tuple of (test_data, ground_truth) - kept for interface compatibility - - Returns: - Tuple of (avg_recall, avg_ndcg, p99_latency, p95_latency) - """ log.info(f"{mp.current_process().name:14} start search the entire test_data to get recall and latency") - - test_data, ground_truth = args - - # Validate input - if not test_data: - raise RuntimeError("empty test_data") - - log.debug(f"test dataset size: {len(test_data)}") - log.debug(f"ground truth size: {len(ground_truth) if ground_truth else 0}") - - # PARALLEL EXECUTION SECTION: - # Create batches for parallel processing - batches = self._create_query_batches() - - # Prepare arguments for each worker process - # Each worker gets: (db_instance, queries, indices, k, filters) - worker_args = [ - (self.db, queries, indices, self.k, self.filters) - for indices, queries in batches - ] - - log.info(f"Launching {len(worker_args)} parallel workers for search") - - # Execute searches in parallel using ProcessPoolExecutor - # Using 'spawn' context ensures clean process initialization - all_results = [] - with concurrent.futures.ProcessPoolExecutor( - max_workers=self.num_workers, - mp_context=mp.get_context("spawn") - ) as executor: - # Submit all batches to worker pool - futures = [ - executor.submit(_parallel_search_worker, args) - for args in worker_args - ] - - # Collect results as workers complete - # as_completed() allows processing results immediately when ready - for future in concurrent.futures.as_completed(futures): + with self.db.init(): + self.db.prepare_filter(self.filters) + test_data, ground_truth = args + ideal_dcg = get_ideal_dcg(self.k) + + log.debug(f"test dataset size: {len(test_data)}") + log.debug(f"ground truth size: {len(ground_truth)}") + + latencies, recalls, ndcgs = [], [], [] + for idx, emb in enumerate(test_data): + s = time.perf_counter() try: - batch_results = future.result() - all_results.extend(batch_results) + results = self._get_db_search_res(emb) except Exception as e: - log.error(f"Worker process failed: {e}") - raise - - # RESULTS PROCESSING SECTION: - # Sort results by original query index to maintain correct ordering - # This is crucial because workers complete in arbitrary order - all_results.sort(key=lambda x: x[1]) # Sort by index (2nd element) - - log.info(f"Collected {len(all_results)} search results from parallel workers") - - # Calculate performance metrics - latencies, recalls, ndcgs = [], [], [] - ideal_dcg = get_ideal_dcg(self.k) - - for latency, idx, results in all_results: - latencies.append(latency) - - # Calculate recall and NDCG if ground truth is available - if ground_truth is not None: - gt = ground_truth[idx] - recalls.append(calc_recall(self.k, gt[:self.k], results)) - ndcgs.append(calc_ndcg(gt[:self.k], results, ideal_dcg)) - else: - recalls.append(0) - ndcgs.append(0) - - # Progress logging every 100 queries - if len(latencies) % 100 == 0: - log.debug( - f"({mp.current_process().name:14}) search_count={len(latencies):3}, " - f"latest_latency={latencies[-1]}, latest recall={recalls[-1]}" - ) - - # Calculate aggregate statistics + log.warning(f"VectorDB search_embedding error: {e}") + raise e from None + + latencies.append(time.perf_counter() - s) + + if ground_truth is not None: + gt = ground_truth[idx] + recalls.append(calc_recall(self.k, gt[: self.k], results)) + ndcgs.append(calc_ndcg(gt[: self.k], results, ideal_dcg)) + else: + recalls.append(0) + ndcgs.append(0) + + if len(latencies) % 100 == 0: + log.debug( + f"({mp.current_process().name:14}) search_count={len(latencies):3}, " + f"latest_latency={latencies[-1]}, latest recall={recalls[-1]}" + ) + avg_latency = round(np.mean(latencies), 4) avg_recall = round(np.mean(recalls), 4) avg_ndcg = round(np.mean(ndcgs), 4) cost = round(np.sum(latencies), 4) p99 = round(np.percentile(latencies, 99), 4) p95 = round(np.percentile(latencies, 95), 4) - log.info( f"{mp.current_process().name:14} search entire test_data: " f"cost={cost}s, " @@ -521,28 +359,15 @@ def search(self, args: tuple[list, list[list[int]]]) -> tuple[float, float, floa f"p99={p99}, " f"p95={p95}" ) - return (avg_recall, avg_ndcg, p99, p95) def _run_in_subprocess(self) -> tuple[float, float, float, float]: - """ - UNCHANGED: Wrapper to execute search in subprocess. - - This method is kept for interface compatibility. - The actual parallelization happens inside the search() method. - """ with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor: future = executor.submit(self.search, (self.test_data, self.ground_truth)) return future.result() @utils.time_it def run(self) -> tuple[float, float, float, float]: - """ - UNCHANGED: Main entry point for search execution. - - Interface remains identical - callers don't need any changes. - Parallel processing is handled transparently. - """ log.info(f"{mp.current_process().name:14} start serial search") if self.test_data is None: msg = "empty test_data" @@ -553,11 +378,9 @@ def run(self) -> tuple[float, float, float, float]: @utils.time_it def run_with_cost(self) -> tuple[tuple[float, float, float, float], float]: """ - UNCHANGED: Search with cost tracking. - + Search all test data in serial. Returns: - tuple[tuple[float, float, float, float], float]: - (avg_recall, avg_ndcg, p99_latency, p95_latency), cost + tuple[tuple[float, float, float, float], float]: (avg_recall, avg_ndcg, p99_latency, p95_latency), cost """ log.info(f"{mp.current_process().name:14} start serial search") if self.test_data is None: @@ -571,13 +394,595 @@ def run_with_cost(self) -> tuple[tuple[float, float, float, float], float]: +# # ======================================================================= + +# # PARALLEL PROCESSING WITH CHECKPOINT FOR RESUME LOGIC + +# # ======================================================================= + + + +# import concurrent +# import logging +# import math +# import multiprocessing as mp +# import time +# import traceback + +# import numpy as np +# import psutil + +# from vectordb_bench.backend.dataset import DatasetManager +# from vectordb_bench.backend.filter import Filter, FilterOp, non_filter + +# from ... import config +# from ...metric import calc_ndcg, calc_recall, get_ideal_dcg +# from ...models import LoadTimeoutError, PerformanceTimeoutError +# from .. import utils +# from ..clients import api + +# import os +# import json + +# # CHECKPOINT_PATH = "/mnt/data/VectorDBBench/insert_checkpoint.json" + +# # Get the path of the current script +# CURRENT_FILE = os.path.abspath(__file__) + +# # Go up 3 levels to reach the root benchmarking folder +# ROOT_BENCHMARK_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(CURRENT_FILE)))) + +# CHECKPOINT_FILENAME = "insert_checkpoint.json" +# CHECKPOINT_PATH = os.path.join(ROOT_BENCHMARK_DIR, CHECKPOINT_FILENAME) + +# # Ensure the root folder exists (should already exist, but safe) +# os.makedirs(ROOT_BENCHMARK_DIR, exist_ok=True) + +# def save_checkpoint(count: int): +# with open(CHECKPOINT_PATH, "w") as f: +# json.dump({"last_index": count}, f) + +# def load_checkpoint() -> int: +# if os.path.exists(CHECKPOINT_PATH): +# try: +# with open(CHECKPOINT_PATH, "r") as f: +# data = json.load(f) +# return data.get("last_index", 0) +# except Exception as e: +# log.warning(f"Failed to read checkpoint file: {e}") +# return 0 + +# def clear_checkpoint(): +# if os.path.exists(CHECKPOINT_PATH): +# os.remove(CHECKPOINT_PATH) + + +# NUM_PER_BATCH = config.NUM_PER_BATCH +# LOAD_MAX_TRY_COUNT = config.LOAD_MAX_TRY_COUNT + +# log = logging.getLogger(__name__) + + +# class SerialInsertRunner: +# def __init__( +# self, +# db: api.VectorDB, +# dataset: DatasetManager, +# normalize: bool, +# filters: Filter = non_filter, +# timeout: float | None = None, +# ): +# self.timeout = timeout if isinstance(timeout, int | float) else None +# self.dataset = dataset +# self.db = db +# self.normalize = normalize +# self.filters = filters + +# def retry_insert(self, db: api.VectorDB, retry_idx: int = 0, **kwargs): +# _, error = db.insert_embeddings(**kwargs) +# if error is not None: +# log.warning(f"Insert Failed, try_idx={retry_idx}, Exception: {error}") +# retry_idx += 1 +# if retry_idx <= config.MAX_INSERT_RETRY: +# time.sleep(retry_idx) +# self.retry_insert(db, retry_idx=retry_idx, **kwargs) +# else: +# msg = f"Insert failed and retried more than {config.MAX_INSERT_RETRY} times" +# raise RuntimeError(msg) from None + +# def task(self) -> int: +# count = 0 +# last_index = load_checkpoint() +# log.info(f"Resuming insertion from checkpoint at index {last_index}") + +# with self.db.init(): +# log.info(f"({mp.current_process().name:16}) Start inserting embeddings in batch {config.NUM_PER_BATCH}") +# start = time.perf_counter() + +# for data_df in self.dataset: +# batch_size = len(data_df) +# if count + batch_size <= last_index: +# count += batch_size +# log.debug(f"Skipping batch of size {batch_size}, total skipped={count}") +# continue + +# all_metadata = data_df[self.dataset.data.train_id_field].tolist() +# emb_np = np.stack(data_df[self.dataset.data.train_vector_field]) + +# if self.normalize: +# log.debug("Normalizing train data") +# all_embeddings = (emb_np / np.linalg.norm(emb_np, axis=1)[:, np.newaxis]).tolist() +# else: +# all_embeddings = emb_np.tolist() +# del emb_np + +# labels_data = None +# if self.filters.type == FilterOp.StrEqual: +# if self.dataset.data.scalar_labels_file_separated: +# labels_data = self.dataset.scalar_labels[self.filters.label_field][all_metadata].to_list() +# else: +# labels_data = data_df[self.filters.label_field].tolist() + +# insert_count, error = self.db.insert_embeddings( +# embeddings=all_embeddings, +# metadata=all_metadata, +# labels_data=labels_data, +# ) + +# if error is not None: +# self.retry_insert( +# self.db, +# embeddings=all_embeddings, +# metadata=all_metadata, +# labels_data=labels_data, +# ) + +# assert insert_count == len(all_metadata) +# count += insert_count +# save_checkpoint(count) # Save progress after each batch + +# if count % 100_000 == 0: +# log.info(f"({mp.current_process().name:16}) Loaded {count} embeddings into VectorDB") + +# log.info( +# f"({mp.current_process().name:16}) Finished loading all dataset into VectorDB, " +# f"dur={time.perf_counter() - start}" +# ) +# clear_checkpoint() # Clean checkpoint file after successful completion +# return count + + +# def endless_insert_data(self, all_embeddings: list, all_metadata: list, left_id: int = 0) -> int: +# with self.db.init(): +# # unique id for endlessness insertion +# all_metadata = [i + left_id for i in all_metadata] + +# num_batches = math.ceil(len(all_embeddings) / NUM_PER_BATCH) +# log.info( +# f"({mp.current_process().name:16}) Start inserting {len(all_embeddings)} " +# f"embeddings in batch {NUM_PER_BATCH}" +# ) +# count = 0 +# for batch_id in range(num_batches): +# retry_count = 0 +# already_insert_count = 0 +# metadata = all_metadata[batch_id * NUM_PER_BATCH : (batch_id + 1) * NUM_PER_BATCH] +# embeddings = all_embeddings[batch_id * NUM_PER_BATCH : (batch_id + 1) * NUM_PER_BATCH] + +# log.debug( +# f"({mp.current_process().name:16}) batch [{batch_id:3}/{num_batches}], " +# f"Start inserting {len(metadata)} embeddings" +# ) +# while retry_count < LOAD_MAX_TRY_COUNT: +# insert_count, error = self.db.insert_embeddings( +# embeddings=embeddings[already_insert_count:], +# metadata=metadata[already_insert_count:], +# ) +# already_insert_count += insert_count +# if error is not None: +# retry_count += 1 +# time.sleep(10) + +# log.info(f"Failed to insert data, try {retry_count} time") +# if retry_count >= LOAD_MAX_TRY_COUNT: +# raise error +# else: +# break +# log.debug( +# f"({mp.current_process().name:16}) batch [{batch_id:3}/{num_batches}], " +# f"Finish inserting {len(metadata)} embeddings" +# ) + +# assert already_insert_count == len(metadata) +# count += already_insert_count +# log.info( +# f"({mp.current_process().name:16}) Finish inserting {len(all_embeddings)} embeddings in " +# f"batch {NUM_PER_BATCH}" +# ) +# return count + +# @utils.time_it +# def _insert_all_batches(self) -> int: +# """Performance case only""" +# with concurrent.futures.ProcessPoolExecutor( +# mp_context=mp.get_context("spawn"), +# max_workers=1, +# ) as executor: +# future = executor.submit(self.task) +# try: +# count = future.result(timeout=self.timeout) +# except TimeoutError as e: +# msg = f"VectorDB load dataset timeout in {self.timeout}" +# log.warning(msg) +# for pid, _ in executor._processes.items(): +# psutil.Process(pid).kill() +# raise PerformanceTimeoutError(msg) from e +# except Exception as e: +# log.warning(f"VectorDB load dataset error: {e}") +# raise e from e +# else: +# return count + +# def run_endlessness(self) -> int: +# """run forever util DB raises exception or crash""" +# # datasets for load tests are quite small, can fit into memory +# # only 1 file +# data_df = next(iter(self.dataset)) +# all_embeddings, all_metadata = ( +# np.stack(data_df[self.dataset.data.train_vector_field]).tolist(), +# data_df[self.dataset.data.train_id_field].tolist(), +# ) + +# start_time = time.perf_counter() +# max_load_count, times = 0, 0 +# try: +# while time.perf_counter() - start_time < self.timeout: +# count = self.endless_insert_data( +# all_embeddings, +# all_metadata, +# left_id=max_load_count, +# ) +# max_load_count += count +# times += 1 +# log.info( +# f"Loaded {times} entire dataset, current max load counts={utils.numerize(max_load_count)}, " +# f"{max_load_count}" +# ) +# except Exception as e: +# log.info( +# f"Capacity case load reach limit, insertion counts={utils.numerize(max_load_count)}, " +# f"{max_load_count}, err={e}" +# ) +# traceback.print_exc() +# return max_load_count +# else: +# raise LoadTimeoutError(self.timeout) + +# def run(self) -> int: +# count, _ = self._insert_all_batches() +# return count + + +# # ============================================================================= +# # PARALLEL SEARCH WORKER FUNCTION +# # ============================================================================= +# def _parallel_search_worker(args: tuple) -> list[tuple[float, int, list[int]]]: +# """ +# Worker function executed in separate process for parallel search. + +# Each worker: +# 1. Receives a batch of queries (embeddings) with their indices +# 2. Initializes its own DB connection +# 3. Executes searches for all queries in the batch +# 4. Returns results with latencies and original indices + +# Args: +# args: Tuple of (db_instance, batch_queries, batch_indices, k, filters) + +# Returns: +# List of tuples: [(latency, original_query_idx, search_results), ...] + +# This function runs in a subprocess, so it needs to: +# - Initialize DB connection independently +# - Handle its own retry logic +# - Return all results for aggregation in main process +# """ +# db_instance, batch_queries, batch_indices, k, filters = args + +# results_batch = [] + +# # Each worker initializes its own DB connection +# with db_instance.init(): +# # Prepare filters if needed (e.g., for filtered search scenarios) +# db_instance.prepare_filter(filters) + +# # Process each query in this worker's batch +# for idx, emb in zip(batch_indices, batch_queries): +# retry_count = 0 +# max_retries = config.MAX_SEARCH_RETRY + +# # Retry logic for each individual query +# while retry_count <= max_retries: +# try: +# # Time the search operation +# s = time.perf_counter() +# results = db_instance.search_embedding(emb, k) +# latency = time.perf_counter() - s + +# # Store result with original index for proper ordering later +# results_batch.append((latency, idx, results)) +# break # Success, move to next query + +# except Exception as e: +# retry_count += 1 +# if retry_count > max_retries: +# log.error(f"Query {idx} failed after {max_retries} retries: {e}") +# raise +# log.warning(f"Query {idx} retry {retry_count}/{max_retries}: {e}") +# time.sleep(0.1 * retry_count) # Exponential backoff + +# return results_batch + + +# class SerialSearchRunner: +# """ +# Search runner that executes queries to measure recall and latency. + +# PERFORMANCE ENHANCEMENT: +# This class now uses parallel processing under the hood while maintaining +# the same interface. Queries are distributed across multiple worker processes +# for significant speedup (4-8x faster depending on CPU cores). + +# The parallelization is transparent to callers - all existing code continues +# to work without modification. +# """ + +# def __init__( +# self, +# db: api.VectorDB, +# test_data: list[list[float]], +# ground_truth: list[list[int]], +# k: int = 100, +# filters: Filter = non_filter, +# ): +# self.db = db +# self.k = k +# self.filters = filters + +# # Convert test data to list format if needed +# if isinstance(test_data[0], np.ndarray): +# self.test_data = [query.tolist() for query in test_data] +# else: +# self.test_data = test_data +# self.ground_truth = ground_truth + +# # PARALLEL PROCESSING CONFIG: +# # Auto-detect number of CPU cores, reserve 1 for main process +# # This allows efficient parallel query execution +# # self.num_workers = max(1, mp.cpu_count() - 1) +# self.num_workers = max(1, 2) +# log.info(f"SerialSearchRunner initialized with {self.num_workers} parallel workers") + +# def _get_db_search_res(self, emb: list[float], retry_idx: int = 0) -> list[int]: +# """ +# LEGACY METHOD - Kept for backward compatibility. +# This method is no longer used in the main search path but retained +# in case any external code calls it directly. +# """ +# try: +# results = self.db.search_embedding(emb, self.k) +# except Exception as e: +# log.warning(f"Serial search failed, retry_idx={retry_idx}, Exception: {e}") +# if retry_idx < config.MAX_SEARCH_RETRY: +# return self._get_db_search_res(emb=emb, retry_idx=retry_idx + 1) + +# msg = f"Serial search failed and retried more than {config.MAX_SEARCH_RETRY} times" +# raise RuntimeError(msg) from e + +# return results + +# def _create_query_batches(self) -> list[tuple[list[int], list]]: +# """ +# PARALLEL PROCESSING HELPER: +# Splits test queries into batches for distribution to worker processes. + +# Each batch contains: +# - Original indices (to maintain ordering after parallel execution) +# - Query embeddings for this batch + +# Batch size is calculated as: total_queries / num_workers +# This ensures even distribution of work across all available workers. + +# Returns: +# List of (indices, queries) tuples, one per worker +# """ +# total_queries = len(self.test_data) +# batch_size = max(1, math.ceil(total_queries / self.num_workers)) +# batches = [] + +# for i in range(0, total_queries, batch_size): +# end_idx = min(i + batch_size, total_queries) +# indices = list(range(i, end_idx)) +# queries = self.test_data[i:end_idx] +# batches.append((indices, queries)) + +# log.debug(f"Created {len(batches)} query batches, batch_size={batch_size}") +# return batches + +# def search(self, args: tuple[list, list[list[int]]]) -> tuple[float, float, float, float]: +# """ +# MODIFIED: Now uses parallel processing for significant speedup. + +# Main changes from original: +# 1. Splits queries into batches for parallel workers +# 2. Spawns worker processes to execute searches concurrently +# 3. Collects and re-orders results by original index +# 4. Calculates metrics (recall, NDCG, latencies) from parallel results + +# The interface and return values remain identical to the original version, +# ensuring backward compatibility with existing code. + +# Original single-threaded approach took ~164s for 1000 queries. +# Parallel approach typically achieves 4-8x speedup depending on CPU cores. + +# Args: +# args: Tuple of (test_data, ground_truth) - kept for interface compatibility + +# Returns: +# Tuple of (avg_recall, avg_ndcg, p99_latency, p95_latency) +# """ +# log.info(f"{mp.current_process().name:14} start search the entire test_data to get recall and latency") + +# test_data, ground_truth = args + +# # Validate input +# if not test_data: +# raise RuntimeError("empty test_data") + +# log.debug(f"test dataset size: {len(test_data)}") +# log.debug(f"ground truth size: {len(ground_truth) if ground_truth else 0}") + +# # PARALLEL EXECUTION SECTION: +# # Create batches for parallel processing +# batches = self._create_query_batches() + +# # Prepare arguments for each worker process +# # Each worker gets: (db_instance, queries, indices, k, filters) +# worker_args = [ +# (self.db, queries, indices, self.k, self.filters) +# for indices, queries in batches +# ] + +# log.info(f"Launching {len(worker_args)} parallel workers for search") + +# # Execute searches in parallel using ProcessPoolExecutor +# # Using 'spawn' context ensures clean process initialization +# all_results = [] +# with concurrent.futures.ProcessPoolExecutor( +# max_workers=self.num_workers, +# mp_context=mp.get_context("spawn") +# ) as executor: +# # Submit all batches to worker pool +# futures = [ +# executor.submit(_parallel_search_worker, args) +# for args in worker_args +# ] + +# # Collect results as workers complete +# # as_completed() allows processing results immediately when ready +# for future in concurrent.futures.as_completed(futures): +# try: +# batch_results = future.result() +# all_results.extend(batch_results) +# except Exception as e: +# log.error(f"Worker process failed: {e}") +# raise + +# # RESULTS PROCESSING SECTION: +# # Sort results by original query index to maintain correct ordering +# # This is crucial because workers complete in arbitrary order +# all_results.sort(key=lambda x: x[1]) # Sort by index (2nd element) + +# log.info(f"Collected {len(all_results)} search results from parallel workers") + +# # Calculate performance metrics +# latencies, recalls, ndcgs = [], [], [] +# ideal_dcg = get_ideal_dcg(self.k) + +# for latency, idx, results in all_results: +# latencies.append(latency) + +# # Calculate recall and NDCG if ground truth is available +# if ground_truth is not None: +# gt = ground_truth[idx] +# recalls.append(calc_recall(self.k, gt[:self.k], results)) +# ndcgs.append(calc_ndcg(gt[:self.k], results, ideal_dcg)) +# else: +# recalls.append(0) +# ndcgs.append(0) + +# # Progress logging every 100 queries +# if len(latencies) % 100 == 0: +# log.debug( +# f"({mp.current_process().name:14}) search_count={len(latencies):3}, " +# f"latest_latency={latencies[-1]}, latest recall={recalls[-1]}" +# ) + +# # Calculate aggregate statistics +# avg_latency = round(np.mean(latencies), 4) +# avg_recall = round(np.mean(recalls), 4) +# avg_ndcg = round(np.mean(ndcgs), 4) +# cost = round(np.sum(latencies), 4) +# p99 = round(np.percentile(latencies, 99), 4) +# p95 = round(np.percentile(latencies, 95), 4) + +# log.info( +# f"{mp.current_process().name:14} search entire test_data: " +# f"cost={cost}s, " +# f"queries={len(latencies)}, " +# f"avg_recall={avg_recall}, " +# f"avg_ndcg={avg_ndcg}, " +# f"avg_latency={avg_latency}, " +# f"p99={p99}, " +# f"p95={p95}" +# ) + +# return (avg_recall, avg_ndcg, p99, p95) + +# def _run_in_subprocess(self) -> tuple[float, float, float, float]: +# """ +# UNCHANGED: Wrapper to execute search in subprocess. + +# This method is kept for interface compatibility. +# The actual parallelization happens inside the search() method. +# """ +# with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor: +# future = executor.submit(self.search, (self.test_data, self.ground_truth)) +# return future.result() + +# @utils.time_it +# def run(self) -> tuple[float, float, float, float]: +# """ +# UNCHANGED: Main entry point for search execution. + +# Interface remains identical - callers don't need any changes. +# Parallel processing is handled transparently. +# """ +# log.info(f"{mp.current_process().name:14} start serial search") +# if self.test_data is None: +# msg = "empty test_data" +# raise RuntimeError(msg) + +# return self._run_in_subprocess() + +# @utils.time_it +# def run_with_cost(self) -> tuple[tuple[float, float, float, float], float]: +# """ +# UNCHANGED: Search with cost tracking. + +# Returns: +# tuple[tuple[float, float, float, float], float]: +# (avg_recall, avg_ndcg, p99_latency, p95_latency), cost +# """ +# log.info(f"{mp.current_process().name:14} start serial search") +# if self.test_data is None: +# msg = "empty test_data" +# raise RuntimeError(msg) + +# return self._run_in_subprocess() + + + + + + -# ======================================================================= +# # ======================================================================= -# MODIFIED TO HAVE CHECKPOINT FOR RESUME LOGIC +# # MODIFIED TO HAVE CHECKPOINT FOR RESUME LOGIC -# ======================================================================= +# # ======================================================================= @@ -960,11 +1365,11 @@ def run_with_cost(self) -> tuple[tuple[float, float, float, float], float]: -# ======================================================================= +# # ======================================================================= -# ORIGINAL SERIAL RUNNER CODE +# # ORIGINAL SERIAL RUNNER CODE -# ======================================================================= +# # ======================================================================= From 621f9c8d78af0392b01cebc475947125cd688f21 Mon Sep 17 00:00:00 2001 From: Mithun Kumar Date: Mon, 29 Dec 2025 09:29:21 +0530 Subject: [PATCH 03/21] Added automated script for the setup in server --- setup_bench.py | 112 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 setup_bench.py diff --git a/setup_bench.py b/setup_bench.py new file mode 100644 index 000000000..f03042409 --- /dev/null +++ b/setup_bench.py @@ -0,0 +1,112 @@ +import subprocess +import sys +import os +import shutil + +def run_command(command, shell=False): + """Runs a shell command and raises an exception on failure.""" + print(f"--> Running: {' '.join(command) if isinstance(command, list) else command}") + try: + subprocess.check_call(command, shell=shell) + except subprocess.CalledProcessError as e: + print(f"Error executing command: {e}") + sys.exit(1) + +def check_and_install_system_deps(): + print("### Checking System Dependencies ###") + + # 1. Update Apt + run_command("sudo apt-get update", shell=True) + + # 2. Check/Install Git + if shutil.which("git") is None: + print("Git not found. Installing...") + run_command("sudo apt-get install -y git", shell=True) + else: + print("Git is already installed.") + + # 3. Check/Install Python 3.11 + # We explicitly check for a python3.11 binary + if shutil.which("python3.11") is None: + print("Python 3.11 not found. Installing via deadsnakes PPA...") + run_command("sudo apt-get install -y software-properties-common", shell=True) + run_command("sudo add-apt-repository -y ppa:deadsnakes/ppa", shell=True) + run_command("sudo apt-get update", shell=True) + run_command("sudo apt-get install -y python3.11 python3.11-venv python3.11-dev", shell=True) + else: + print("Python 3.11 is already installed.") + +def setup_repo(): + repo_url = "https://github.com/EndeeLabs/VectorDBBench.git" + repo_dir = "VectorDBBench" + + print("\n### Setting up Repository ###") + + # 1. Clone + if not os.path.exists(repo_dir): + run_command(["git", "clone", repo_url]) + else: + print(f"Directory {repo_dir} already exists. Skipping clone.") + + # Change working directory to the repo + os.chdir(repo_dir) + + # 2. Switch Branch + print("Switching to 'Endee' branch...") + run_command(["git", "fetch", "origin"]) + run_command(["git", "checkout", "Endee"]) + + # Pull latest just in case + run_command(["git", "pull", "origin", "Endee"]) + + return os.getcwd() + +def setup_venv_and_install(repo_path): + print("\n### Setting up Virtual Environment & Installing Packages ###") + + venv_dir = os.path.join(repo_path, "venv") + + # 1. Create Venv using Python 3.11 specifically + if not os.path.exists(venv_dir): + print("Creating virtual environment...") + run_command(["python3.11", "-m", "venv", "venv"]) + else: + print("Virtual environment already exists.") + + # Define paths to the venv executables + venv_python = os.path.join(venv_dir, "bin", "python") + venv_pip = os.path.join(venv_dir, "bin", "pip") + + # Upgrade pip first + run_command([venv_python, "-m", "pip", "install", "--upgrade", "pip"]) + + # 2. pip install endee + print("Installing 'endee'...") + run_command([venv_pip, "install", "endee"]) + + # 3. pip install -e . + print("Installing editable project...") + run_command([venv_pip, "install", "-e", "."]) + + return venv_dir + +if __name__ == "__main__": + # Ensure we are on Linux (quick check) + if not sys.platform.startswith('linux'): + print("Warning: This script is optimized for Linux (AWS/GCP instances).") + + # Step 1: System Level Installs (Sudo required) + check_and_install_system_deps() + + # Step 2: Clone and Switch Branch + project_path = setup_repo() + + # Step 3 & 4: Create Venv and Install Pip packages + venv_path = setup_venv_and_install(project_path) + + print("\n" + "="*50) + print("SETUP COMPLETE!") + print("="*50) + print("To start using VectorDBBench, run the following command in your shell:") + print(f"\n source {os.path.join(project_path, 'venv/bin/activate')}\n") + print("="*50) \ No newline at end of file From 4da6487a0a32987c4b6fcab0b9215ad421c9aea0 Mon Sep 17 00:00:00 2001 From: Mithun Kumar Date: Mon, 29 Dec 2025 10:29:06 +0530 Subject: [PATCH 04/21] Added small precision for cli --- vectordb_bench/backend/clients/endee/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vectordb_bench/backend/clients/endee/cli.py b/vectordb_bench/backend/clients/endee/cli.py index e6096fc2e..597ea9b4c 100644 --- a/vectordb_bench/backend/clients/endee/cli.py +++ b/vectordb_bench/backend/clients/endee/cli.py @@ -36,7 +36,7 @@ class EndeeTypedDict(CommonTypedDict): ] precision: Annotated[ str, - click.option("--precision", type=click.Choice(["medium", "high", "ultra-high", "fp16"]), default="medium", help="Quant Level", show_default=True) + click.option("--precision", type=click.Choice(["medium", "high", "ultra-high", "fp16", "small"]), default="medium", help="Quant Level", show_default=True) ] version: Annotated[ int, From d04d1eb5c8eed166fa5521df0a00bcaa7eeb1006 Mon Sep 17 00:00:00 2001 From: Mithun Kumar Date: Mon, 29 Dec 2025 10:35:11 +0530 Subject: [PATCH 05/21] Modified demoCommands file --- .allcommands => .demoCommands | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) rename .allcommands => .demoCommands (87%) diff --git a/.allcommands b/.demoCommands similarity index 87% rename from .allcommands rename to .demoCommands index 355251929..38717a123 100644 --- a/.allcommands +++ b/.demoCommands @@ -1,6 +1,6 @@ # FOR CREATING A NEW INDEX AND START THE PROCESS DATASET_LOCAL_DIR="/home/shaleen/vectordataset" vectordbbench endee \ - --token "as1prod:nb5l9VEBrgDEsvjut8Oawgcbf5jghoN5" \ + --token "TOKEN" \ --region india-west-1 \ --base-url "https://as1.endee.io/api/v1" \ --index-name test_prod_10M_medium_3 \ @@ -21,11 +21,10 @@ DATASET_LOCAL_DIR="/home/shaleen/vectordataset" vectordbbench endee \ --search-concurrent \ --search-serial -PROD: "https://as1.endee.io/api/v1" -AWS: http://54.89.169.48:80/api/v1 +======================================================================== DATASET_LOCAL_DIR="/home/shaleen/vectordataset" vectordbbench endee \ - --token "awsbench16:uVQMSo85De3k1lN5FiZI0xKcwXDlyBvu" \ + --token "TOKEN" \ --region india-west-1 \ --base-url "http://54.89.169.48:80/api/v1"\ --index-name test_aws_10M_medium_1 \ @@ -46,9 +45,11 @@ AWS: http://54.89.169.48:80/api/v1 --search-concurrent \ --search-serial +======================================================================== + # FOR USING THE EXISTING INDEX DATASET_LOCAL_DIR="/home/shaleen/vectordataset" vectordbbench endee \ - --token "as1prod:nb5l9VEBrgDEsvjut8Oawgcbf5jghoN5" \ + --token "TOKEN" \ --region india-west-1 \ --base-url "https://as1.endee.io/api/v1" \ --index-name test_prod_10M_medium_3 \ From a6304ff680dfb49223e791e1532c980eb8eab11f Mon Sep 17 00:00:00 2001 From: Mithun Kumar Date: Mon, 29 Dec 2025 11:17:04 +0530 Subject: [PATCH 06/21] Added download dataset file --- download_dataset.py | 177 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 download_dataset.py diff --git a/download_dataset.py b/download_dataset.py new file mode 100644 index 000000000..3fce58cde --- /dev/null +++ b/download_dataset.py @@ -0,0 +1,177 @@ +# =================================================================== # +# ==================== FOR COHERE 10M DATASET ======================= # +# =================================================================== # + +import os +import requests +from concurrent.futures import ThreadPoolExecutor +from tqdm import tqdm + +# --- CONFIGURATION --- +# Base URL for Cohere 10M dataset +BASE_URL = "https://assets.zilliz.com/benchmark/cohere_large_10m/" + +# Your SSD Path (Updated based on your terminal output) +LOCAL_DIR = "/home/admin/vectordataset/cohere/cohere_large_10m" + +# Number of parallel downloads +MAX_WORKERS = 10 + +# --- FILE LIST GENERATION --- +files = [ + "test.parquet", + "neighbors.parquet", + "scalar_labels.parquet" # Included based on your ls output +] + +# Generate the 10 training parts (00 to 09) +# Note: Cohere 10M uses the prefix 'shuffle_train-' and has 10 parts +for i in range(10): + files.append(f"shuffle_train-{i:02d}-of-10.parquet") + +def download_file(filename): + local_path = os.path.join(LOCAL_DIR, filename) + url = BASE_URL + filename + + # 1. Skip if fully downloaded (File > 1KB) + if os.path.exists(local_path) and os.path.getsize(local_path) > 1024: + return f"✅ Skipped (Exists): {filename}" + + try: + # 2. Stream download + with requests.get(url, stream=True, timeout=60) as r: + if r.status_code == 404: + return f"❌ Missing (404): {filename}" + + r.raise_for_status() + + # Write to file + with open(local_path, 'wb') as f: + for chunk in r.iter_content(chunk_size=1024*1024): # 1MB chunks + f.write(chunk) + + return f"⬇️ Downloaded: {filename}" + + except Exception as e: + return f"⚠️ Error {filename}: {str(e)}" + +if __name__ == "__main__": + # Ensure directory exists + os.makedirs(LOCAL_DIR, exist_ok=True) + + print(f"--- Cohere 10M Downloader ---") + print(f"Source: {BASE_URL}") + print(f"Destination: {LOCAL_DIR}") + print(f"Files to check: {len(files)}") + print(f"Parallel threads: {MAX_WORKERS}\n") + + # Start Parallel Download + with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: + # We use tqdm to show a progress bar for the *number of files* completed + results = list(tqdm(executor.map(download_file, files), total=len(files), unit="file")) + + # Print summary of missing or failed files + failures = [r for r in results if "Error" in r or "Missing" in r] + if failures: + print("\nSummary of Issues:") + for fail in failures: + print(fail) + else: + print("\n🎉 All files processed successfully!") + + + + + + + + + + +# # =================================================================== # +# # ==================== FOR LAION 100M DATASET ======================= # +# # =================================================================== # + +# import os +# import requests +# from concurrent.futures import ThreadPoolExecutor +# from tqdm import tqdm + +# # --- CONFIGURATION --- +# # Base URL for Laion 100M dataset +# BASE_URL = "https://assets.zilliz.com/benchmark/laion_large_100m/" + +# # Your SSD Path (Updated based on your terminal output) +# LOCAL_DIR = "/home/admin/vectordataset/laion/laion_large_100m" + +# # Number of parallel downloads +# MAX_WORKERS = 10 + +# # --- FILE LIST GENERATION --- +# files = [ +# "test.parquet", +# "neighbors.parquet", +# # "scalar_labels.parquet" # Uncomment if needed (often missing for 100M) +# ] + +# # Generate the 100 training parts (00 to 99) +# for i in range(100): +# files.append(f"train-{i:02d}-of-100.parquet") + + +# def download_file(filename): +# local_path = os.path.join(LOCAL_DIR, filename) +# url = BASE_URL + filename + +# # 1. Skip if fully downloaded (File > 1KB) +# if os.path.exists(local_path) and os.path.getsize(local_path) > 1024: +# return f"✅ Skipped (Exists): {filename}" + +# try: +# # 2. Stream download +# with requests.get(url, stream=True, timeout=60) as r: +# if r.status_code == 404: +# return f"❌ Missing (404): {filename}" + +# r.raise_for_status() + +# # Write to file +# with open(local_path, 'wb') as f: +# for chunk in r.iter_content(chunk_size=1024 * 1024): # 1MB chunks +# f.write(chunk) + +# return f"⬇️ Downloaded: {filename}" + +# except Exception as e: +# return f"⚠️ Error {filename}: {str(e)}" + + +# if __name__ == "__main__": +# # Ensure directory exists +# os.makedirs(LOCAL_DIR, exist_ok=True) + +# print(f"--- LAION 100M Downloader ---") +# print(f"Source: {BASE_URL}") +# print(f"Destination: {LOCAL_DIR}") +# print(f"Files to check: {len(files)}") +# print(f"Parallel threads: {MAX_WORKERS}\n") + +# # Start Parallel Download +# with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: +# # We use tqdm to show a progress bar for the *number of files* completed +# results = list( +# tqdm( +# executor.map(download_file, files), +# total=len(files), +# unit="file" +# ) +# ) + +# # Print summary of missing or failed files +# failures = [r for r in results if "Error" in r or "Missing" in r] +# if failures: +# print("\nSummary of Issues:") +# for fail in failures: +# print(fail) +# else: +# print("\n🎉 All files processed successfully!") From 1d1912d482f92377bcfbde6ee17e67c6a281cda8 Mon Sep 17 00:00:00 2001 From: Mithun Kumar Date: Mon, 29 Dec 2025 12:08:51 +0530 Subject: [PATCH 07/21] Added Demo CLI Commands file --- .demoCommands => demo_cli_ommands.txt | 36 ++++++--------------------- 1 file changed, 8 insertions(+), 28 deletions(-) rename .demoCommands => demo_cli_ommands.txt (51%) diff --git a/.demoCommands b/demo_cli_ommands.txt similarity index 51% rename from .demoCommands rename to demo_cli_ommands.txt index 38717a123..4eea84e6d 100644 --- a/.demoCommands +++ b/demo_cli_ommands.txt @@ -1,9 +1,11 @@ +# ======================================================================== # FOR CREATING A NEW INDEX AND START THE PROCESS -DATASET_LOCAL_DIR="/home/shaleen/vectordataset" vectordbbench endee \ +# ======================================================================== +DATASET_LOCAL_DIR="/home/admin/vectordataset" vectordbbench endee \ --token "TOKEN" \ --region india-west-1 \ --base-url "https://as1.endee.io/api/v1" \ - --index-name test_prod_10M_medium_3 \ + --index-name test_index_name \ --task-label "20251224" \ --m 16 \ --ef-con 128 \ @@ -21,38 +23,16 @@ DATASET_LOCAL_DIR="/home/shaleen/vectordataset" vectordbbench endee \ --search-concurrent \ --search-serial -======================================================================== - DATASET_LOCAL_DIR="/home/shaleen/vectordataset" vectordbbench endee \ - --token "TOKEN" \ - --region india-west-1 \ - --base-url "http://54.89.169.48:80/api/v1"\ - --index-name test_aws_10M_medium_1 \ - --task-label "20251223" \ - --m 16 \ - --ef-con 128 \ - --ef-search 128 \ - --space-type cosine \ - --precision medium \ - --version 1 \ - --case-type Performance768D10M \ - --k 30 \ - --num-concurrency "8" \ - --concurrency-duration 30 \ - --concurrency-timeout 3600 \ - --drop-old \ - --load \ - --search-concurrent \ - --search-serial - -======================================================================== +# ======================================================================== # FOR USING THE EXISTING INDEX -DATASET_LOCAL_DIR="/home/shaleen/vectordataset" vectordbbench endee \ +# ======================================================================== +DATASET_LOCAL_DIR="/home/admin/vectordataset" vectordbbench endee \ --token "TOKEN" \ --region india-west-1 \ --base-url "https://as1.endee.io/api/v1" \ - --index-name test_prod_10M_medium_3 \ + --index-name test_index_name\ --task-label "20251224" \ --m 16 \ --ef-con 128 \ From 1599f221bfd21d9d2d7b1b94928cacefb439ef10 Mon Sep 17 00:00:00 2001 From: Mithun Kumar Date: Mon, 29 Dec 2025 12:36:03 +0530 Subject: [PATCH 08/21] Modified the selecting options --- download_dataset.py | 209 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 169 insertions(+), 40 deletions(-) diff --git a/download_dataset.py b/download_dataset.py index 3fce58cde..cc04abd90 100644 --- a/download_dataset.py +++ b/download_dataset.py @@ -1,40 +1,72 @@ -# =================================================================== # -# ==================== FOR COHERE 10M DATASET ======================= # -# =================================================================== # - import os import requests from concurrent.futures import ThreadPoolExecutor from tqdm import tqdm -# --- CONFIGURATION --- -# Base URL for Cohere 10M dataset -BASE_URL = "https://assets.zilliz.com/benchmark/cohere_large_10m/" - -# Your SSD Path (Updated based on your terminal output) -LOCAL_DIR = "/home/admin/vectordataset/cohere/cohere_large_10m" - -# Number of parallel downloads -MAX_WORKERS = 10 - -# --- FILE LIST GENERATION --- -files = [ - "test.parquet", - "neighbors.parquet", - "scalar_labels.parquet" # Included based on your ls output -] +# =================================================================== +# ======================== USER CONFIGURATION ======================= + +# 1. SELECT DATASET +DATASET_NAME = "cohere" # Options: "cohere", "laion" + +# 2. ROOT DIRECTORY +ROOT_DIR = "/home/admin/vectordataset" + +# =================================================================== + +# 3. DATASET DETAILS +DATASET_CONFIG = { + "cohere": { + "base_url": "https://assets.zilliz.com/benchmark/cohere_large_10m/", + "rel_path": "cohere/cohere_large_10m", # Subfolder inside ROOT_DIR + "train_prefix": "shuffle_train", + "train_count": 10, + "train_suffix": "of-10.parquet", + "extra_files": ["scalar_labels.parquet"] + }, + "laion": { + "base_url": "https://assets.zilliz.com/benchmark/laion_large_100m/", + "rel_path": "laion/laion_large_100m", # Subfolder inside ROOT_DIR + "train_prefix": "train", + "train_count": 100, + "train_suffix": "of-100.parquet", + "extra_files": [] + } +} + +# 4. GLOBAL SETTINGS +MAX_WORKERS = 10 + + +def get_file_list(config): + """Generates the list of files based on the dataset config.""" + files = ["test.parquet", "neighbors.parquet"] + + # Add extra files (like scalar_labels) if they exist in config + files.extend(config["extra_files"]) + + # Generate Training Files + # Example: shuffle_train-00-of-10.parquet OR train-00-of-100.parquet + prefix = config["train_prefix"] + count = config["train_count"] + suffix = config["train_suffix"] + + for i in range(count): + # Format usually needs leading zeros (00, 01..99) + files.append(f"{prefix}-{i:02d}-{suffix}") + + return files -# Generate the 10 training parts (00 to 09) -# Note: Cohere 10M uses the prefix 'shuffle_train-' and has 10 parts -for i in range(10): - files.append(f"shuffle_train-{i:02d}-of-10.parquet") +def download_file(args): + """ + Downloads a single file. args is (filename, base_url, full_local_path) + """ + filename, base_url, dest_dir = args + local_file_path = os.path.join(dest_dir, filename) + url = base_url + filename -def download_file(filename): - local_path = os.path.join(LOCAL_DIR, filename) - url = BASE_URL + filename - # 1. Skip if fully downloaded (File > 1KB) - if os.path.exists(local_path) and os.path.getsize(local_path) > 1024: + if os.path.exists(local_file_path) and os.path.getsize(local_file_path) > 1024: return f"✅ Skipped (Exists): {filename}" try: @@ -46,7 +78,7 @@ def download_file(filename): r.raise_for_status() # Write to file - with open(local_path, 'wb') as f: + with open(local_file_path, 'wb') as f: for chunk in r.iter_content(chunk_size=1024*1024): # 1MB chunks f.write(chunk) @@ -56,21 +88,36 @@ def download_file(filename): return f"⚠️ Error {filename}: {str(e)}" if __name__ == "__main__": + if DATASET_NAME not in DATASET_CONFIG: + print(f"❌ Error: Invalid dataset name '{DATASET_NAME}'") + exit(1) + + cfg = DATASET_CONFIG[DATASET_NAME] + + # Construct the full destination path + full_dest_path = os.path.join(ROOT_DIR, cfg["rel_path"]) + # Ensure directory exists - os.makedirs(LOCAL_DIR, exist_ok=True) + os.makedirs(full_dest_path, exist_ok=True) - print(f"--- Cohere 10M Downloader ---") - print(f"Source: {BASE_URL}") - print(f"Destination: {LOCAL_DIR}") - print(f"Files to check: {len(files)}") - print(f"Parallel threads: {MAX_WORKERS}\n") + files_to_download = get_file_list(cfg) + + print(f"--- Unified Downloader: {DATASET_NAME.upper()} ---") + print(f"Source: {cfg['base_url']}") + print(f"Destination: {full_dest_path}") + print(f"File Count: {len(files_to_download)}") + print(f"Threads: {MAX_WORKERS}\n") - # Start Parallel Download + # Pack arguments: (filename, base_url, full_dest_path) + map_args = [(f, cfg["base_url"], full_dest_path) for f in files_to_download] + with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: - # We use tqdm to show a progress bar for the *number of files* completed - results = list(tqdm(executor.map(download_file, files), total=len(files), unit="file")) + results = list(tqdm( + executor.map(download_file, map_args), + total=len(files_to_download), + unit="file" + )) - # Print summary of missing or failed files failures = [r for r in results if "Error" in r or "Missing" in r] if failures: print("\nSummary of Issues:") @@ -84,6 +131,88 @@ def download_file(filename): +# # =================================================================== # +# # ==================== FOR COHERE 10M DATASET ======================= # +# # =================================================================== # + +# import os +# import requests +# from concurrent.futures import ThreadPoolExecutor +# from tqdm import tqdm + +# # --- CONFIGURATION --- +# # Base URL for Cohere 10M dataset +# BASE_URL = "https://assets.zilliz.com/benchmark/cohere_large_10m/" + +# # Your SSD Path (Updated based on your terminal output) +# LOCAL_DIR = "/home/admin/vectordataset/cohere/cohere_large_10m" + +# # Number of parallel downloads +# MAX_WORKERS = 10 + +# # --- FILE LIST GENERATION --- +# files = [ +# "test.parquet", +# "neighbors.parquet", +# "scalar_labels.parquet" # Included based on your ls output +# ] + +# # Generate the 10 training parts (00 to 09) +# # Note: Cohere 10M uses the prefix 'shuffle_train-' and has 10 parts +# for i in range(10): +# files.append(f"shuffle_train-{i:02d}-of-10.parquet") + +# def download_file(filename): +# local_path = os.path.join(LOCAL_DIR, filename) +# url = BASE_URL + filename + +# # 1. Skip if fully downloaded (File > 1KB) +# if os.path.exists(local_path) and os.path.getsize(local_path) > 1024: +# return f"✅ Skipped (Exists): {filename}" + +# try: +# # 2. Stream download +# with requests.get(url, stream=True, timeout=60) as r: +# if r.status_code == 404: +# return f"❌ Missing (404): {filename}" + +# r.raise_for_status() + +# # Write to file +# with open(local_path, 'wb') as f: +# for chunk in r.iter_content(chunk_size=1024*1024): # 1MB chunks +# f.write(chunk) + +# return f"⬇️ Downloaded: {filename}" + +# except Exception as e: +# return f"⚠️ Error {filename}: {str(e)}" + +# if __name__ == "__main__": +# # Ensure directory exists +# os.makedirs(LOCAL_DIR, exist_ok=True) + +# print(f"--- Cohere 10M Downloader ---") +# print(f"Source: {BASE_URL}") +# print(f"Destination: {LOCAL_DIR}") +# print(f"Files to check: {len(files)}") +# print(f"Parallel threads: {MAX_WORKERS}\n") + +# # Start Parallel Download +# with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: +# # We use tqdm to show a progress bar for the *number of files* completed +# results = list(tqdm(executor.map(download_file, files), total=len(files), unit="file")) + +# # Print summary of missing or failed files +# failures = [r for r in results if "Error" in r or "Missing" in r] +# if failures: +# print("\nSummary of Issues:") +# for fail in failures: +# print(fail) +# else: +# print("\n🎉 All files processed successfully!") + + From b9c2e4b8256b0f9c2045d51b28665b11bd0557de Mon Sep 17 00:00:00 2001 From: Mithun Kumar Date: Tue, 30 Dec 2025 17:04:22 +0530 Subject: [PATCH 09/21] Added new parameter for CLI command --- demo_cli_ommands.txt | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/demo_cli_ommands.txt b/demo_cli_ommands.txt index 4eea84e6d..48001e1f5 100644 --- a/demo_cli_ommands.txt +++ b/demo_cli_ommands.txt @@ -49,3 +49,29 @@ DATASET_LOCAL_DIR="/home/admin/vectordataset" vectordbbench endee \ --skip-load \ --search-concurrent \ --search-serial + + +# ======================================================================== +# WITH CHANGED BATCH SIZE, CREATING A NEW INDEX AND START THE PROCESS +# ======================================================================== +NUM_PER_BATCH=10000 DATASET_LOCAL_DIR="/home/admin/vectordataset" vectordbbench endee \ + --token "TOKEN" \ + --region india-west-1 \ + --base-url "https://as1.endee.io/api/v1" \ + --index-name test_index_name \ + --task-label "20251224" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision medium \ + --version 1 \ + --case-type Performance768D10M \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial From 71ddffd1e9b162da72e8eaf3f64306621ec685bc Mon Sep 17 00:00:00 2001 From: Mithun Kumar Date: Tue, 6 Jan 2026 14:08:38 +0530 Subject: [PATCH 10/21] Updated precision types --- vectordb_bench/backend/clients/endee/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vectordb_bench/backend/clients/endee/cli.py b/vectordb_bench/backend/clients/endee/cli.py index 597ea9b4c..970e4ce7a 100644 --- a/vectordb_bench/backend/clients/endee/cli.py +++ b/vectordb_bench/backend/clients/endee/cli.py @@ -36,7 +36,8 @@ class EndeeTypedDict(CommonTypedDict): ] precision: Annotated[ str, - click.option("--precision", type=click.Choice(["medium", "high", "ultra-high", "fp16", "small"]), default="medium", help="Quant Level", show_default=True) + # click.option("--precision", type=click.Choice(["medium", "high", "ultra-high", "fp16", "small"]), default="medium", help="Quant Level", show_default=True) + click.option("--precision", type=click.Choice(["binary", "int4d", "int8d", "int16d", "float16", "float32"]), default="int8d", help="Quant Level", show_default=True) ] version: Annotated[ int, From a921498b22b7d9181bedd57a53f417cd241b2518 Mon Sep 17 00:00:00 2001 From: Mithun Kumar Date: Tue, 6 Jan 2026 20:57:48 +0530 Subject: [PATCH 11/21] Modified setup bench script --- setup_bench.py | 183 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 182 insertions(+), 1 deletion(-) diff --git a/setup_bench.py b/setup_bench.py index f03042409..1353f4c76 100644 --- a/setup_bench.py +++ b/setup_bench.py @@ -3,6 +3,185 @@ import os import shutil +# ================= CONFIGURATION ================= +REPO_URL = "https://github.com/EndeeLabs/VectorDBBench.git" +REPO_DIR = "VectorDBBench" +PYTHON_VERSION = "3.11.9" +# ================================================= + +def run_command(command, shell=False, cwd=None): + """Runs a shell command and exits on failure.""" + cmd_str = ' '.join(command) if isinstance(command, list) else command + print(f"--> [EXEC]: {cmd_str}") + try: + subprocess.check_call(command, shell=shell, cwd=cwd) + except subprocess.CalledProcessError as e: + print(f"Error executing command: {e}") + sys.exit(1) + +def is_python311_installed(): + """Checks if python3.11 is currently available in the system PATH.""" + # Check standard PATH + if shutil.which("python3.11") is not None: + return True + + # Check common local install location (Source builds often go here) + if os.path.exists("/usr/local/bin/python3.11"): + return True + + return False + +def check_system_compatibility(): + """Ensures we are on a Debian-based system (Debian, Ubuntu, Mint, Kali, etc).""" + if shutil.which("apt-get") is None: + print("\n!!! CRITICAL ERROR !!!") + print("This script relies on 'apt-get'. It works on Debian, Ubuntu, Linux Mint, Kali, Pop!_OS, etc.") + sys.exit(1) + +def is_ubuntu(): + """Returns True only if we are confident this is Ubuntu.""" + try: + if os.path.exists("/etc/os-release"): + with open("/etc/os-release") as f: + if "ubuntu" in f.read().lower(): + return True + if shutil.which("lsb_release"): + out = subprocess.check_output(["lsb_release", "-i"]).decode().lower() + if "ubuntu" in out: + return True + except: + pass + return False + +def install_python_ubuntu_strategy(): + """Strategy A: Ubuntu (Fast PPA)""" + print("\n[Strategy] Detected Ubuntu. Using fast PPA installation...") + run_command("sudo apt-get update", shell=True) + run_command("sudo apt-get install -y software-properties-common git", shell=True) + print("Adding Deadsnakes PPA...") + run_command("sudo add-apt-repository -y ppa:deadsnakes/ppa", shell=True) + run_command("sudo apt-get update", shell=True) + run_command("sudo apt-get install -y python3.11 python3.11-venv python3.11-dev", shell=True) + +def install_python_debian_strategy(): + """Strategy B: Debian / Universal (Source Build Fallback)""" + print("\n[Strategy] Detected Debian/Other. Using robust compatibility mode...") + + # 1. Install Dependencies + run_command("sudo apt-get update", shell=True) + print("Installing build dependencies...") + deps = [ + "git", "wget", "build-essential", "zlib1g-dev", "libncurses5-dev", + "libgdbm-dev", "libnss3-dev", "libssl-dev", "libreadline-dev", + "libffi-dev", "libsqlite3-dev", "libbz2-dev", "pkg-config" + ] + run_command(f"sudo apt-get install -y {' '.join(deps)}", shell=True) + + # 2. Try APT first (Debian 12+) + print("Checking system repos for Python 3.11...") + try: + subprocess.check_call("apt-cache show python3.11", shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + print("Python 3.11 found in APT. Installing...") + run_command("sudo apt-get install -y python3.11 python3.11-venv python3.11-dev", shell=True) + return + except subprocess.CalledProcessError: + print("Python 3.11 not in system repos. Proceeding to Source Build.") + + # 3. Source Build (Universal) + print("\n*** STARTING SOURCE BUILD ***") + tarball = f"Python-{PYTHON_VERSION}.tgz" + url = f"https://www.python.org/ftp/python/{PYTHON_VERSION}/{tarball}" + + if not os.path.exists(tarball): + run_command(f"wget {url}", shell=True) + + run_command(f"tar -xf {tarball}", shell=True) + src_dir = f"Python-{PYTHON_VERSION}" + + # Configure & Make + run_command("./configure --enable-optimizations", shell=True, cwd=src_dir) + nproc = subprocess.check_output("nproc", shell=True).decode().strip() + run_command(f"make -j {nproc}", shell=True, cwd=src_dir) + + # SAFE INSTALL (altinstall) + run_command("sudo make altinstall", shell=True, cwd=src_dir) + + # Cleanup + os.remove(tarball) + run_command(f"sudo rm -rf {src_dir}", shell=True) + +def setup_project_and_venv(): + print("\n[Project] Setting up VectorDBBench...") + + # 1. Clone + if not os.path.exists(REPO_DIR): + run_command(["git", "clone", REPO_URL]) + + os.chdir(REPO_DIR) + + # 2. Switch Branch + run_command(["git", "fetch", "origin"]) + run_command(["git", "checkout", "Endee"]) + run_command(["git", "pull", "origin", "Endee"]) + + # 3. Locate Python 3.11 + python_bin = "python3.11" + if shutil.which("python3.11") is None: + if os.path.exists("/usr/local/bin/python3.11"): + python_bin = "/usr/local/bin/python3.11" + else: + print("Error: Python 3.11 binary not found after installation attempt.") + sys.exit(1) + + print(f"Using Python binary: {python_bin}") + + # 4. Create Venv + if not os.path.exists("venv"): + run_command([python_bin, "-m", "venv", "venv"]) + else: + print("Virtual environment already exists.") + + # 5. Install Deps + venv_pip = os.path.join("venv", "bin", "pip") + run_command([venv_pip, "install", "--upgrade", "pip"]) + run_command([venv_pip, "install", "endee"]) + run_command([venv_pip, "install", "-e", "."]) + + return os.path.join(os.getcwd(), "venv") + +if __name__ == "__main__": + check_system_compatibility() + + # --- THE FIX IS HERE --- + if is_python311_installed(): + print("\n" + "="*50) + print("SKIP: Python 3.11 is already installed.") + print("="*50) + else: + # Only install if missing + if is_ubuntu(): + install_python_ubuntu_strategy() + else: + install_python_debian_strategy() + # ----------------------- + + venv_path = setup_project_and_venv() + + print("\n" + "="*50) + print("SETUP SUCCESSFUL!") + print(f"To start: source {os.path.join(venv_path, 'bin', 'activate')}") + print("="*50) + + + + + +''' +import subprocess +import sys +import os +import shutil + def run_command(command, shell=False): """Runs a shell command and raises an exception on failure.""" print(f"--> Running: {' '.join(command) if isinstance(command, list) else command}") @@ -109,4 +288,6 @@ def setup_venv_and_install(repo_path): print("="*50) print("To start using VectorDBBench, run the following command in your shell:") print(f"\n source {os.path.join(project_path, 'venv/bin/activate')}\n") - print("="*50) \ No newline at end of file + print("="*50) + +''' \ No newline at end of file From 737b11a4d02de9d3c7527ff26eee98c39a04c9ac Mon Sep 17 00:00:00 2001 From: Mithun Kumar Date: Wed, 7 Jan 2026 12:30:13 +0530 Subject: [PATCH 12/21] Modified demo cli commands --- demo_cli_ommands.txt | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/demo_cli_ommands.txt b/demo_cli_ommands.txt index 48001e1f5..a770786bc 100644 --- a/demo_cli_ommands.txt +++ b/demo_cli_ommands.txt @@ -4,16 +4,16 @@ DATASET_LOCAL_DIR="/home/admin/vectordataset" vectordbbench endee \ --token "TOKEN" \ --region india-west-1 \ - --base-url "https://as1.endee.io/api/v1" \ + --base-url "http://127.0.0.1:8080/api/v1" \ --index-name test_index_name \ - --task-label "20251224" \ + --task-label "20260107" \ --m 16 \ --ef-con 128 \ --ef-search 128 \ --space-type cosine \ - --precision medium \ + --precision int8d \ --version 1 \ - --case-type Performance768D10M \ + --case-type Performance768D1M \ --k 30 \ --num-concurrency "8" \ --concurrency-duration 30 \ @@ -31,16 +31,16 @@ DATASET_LOCAL_DIR="/home/admin/vectordataset" vectordbbench endee \ DATASET_LOCAL_DIR="/home/admin/vectordataset" vectordbbench endee \ --token "TOKEN" \ --region india-west-1 \ - --base-url "https://as1.endee.io/api/v1" \ + --base-url "http://127.0.0.1:8080/api/v1" \ --index-name test_index_name\ - --task-label "20251224" \ + --task-label "20260107" \ --m 16 \ --ef-con 128 \ --ef-search 128 \ --space-type cosine \ - --precision medium \ + --precision int8d \ --version 1 \ - --case-type Performance768D10M \ + --case-type Performance768D1M \ --k 30 \ --num-concurrency "8" \ --concurrency-duration 30 \ @@ -57,16 +57,16 @@ DATASET_LOCAL_DIR="/home/admin/vectordataset" vectordbbench endee \ NUM_PER_BATCH=10000 DATASET_LOCAL_DIR="/home/admin/vectordataset" vectordbbench endee \ --token "TOKEN" \ --region india-west-1 \ - --base-url "https://as1.endee.io/api/v1" \ + --base-url "http://127.0.0.1:8080/api/v1" \ --index-name test_index_name \ - --task-label "20251224" \ + --task-label "20260107" \ --m 16 \ --ef-con 128 \ --ef-search 128 \ --space-type cosine \ - --precision medium \ + --precision int8d \ --version 1 \ - --case-type Performance768D10M \ + --case-type Performance768D1M \ --k 30 \ --num-concurrency "8" \ --concurrency-duration 30 \ From 0aff8597b959a48e88028ebcc0cf8ba4985f6ead Mon Sep 17 00:00:00 2001 From: Mithun Kumar Date: Wed, 7 Jan 2026 13:03:03 +0530 Subject: [PATCH 13/21] Modified setup bench for different OS --- setup_bench.py | 242 ++++++++++++++++++++++++++++++------------------- 1 file changed, 151 insertions(+), 91 deletions(-) diff --git a/setup_bench.py b/setup_bench.py index 1353f4c76..ad41055c1 100644 --- a/setup_bench.py +++ b/setup_bench.py @@ -1,7 +1,9 @@ -import subprocess import sys import os +import subprocess import shutil +import platform +import urllib.request # ================= CONFIGURATION ================= REPO_URL = "https://github.com/EndeeLabs/VectorDBBench.git" @@ -11,107 +13,145 @@ def run_command(command, shell=False, cwd=None): """Runs a shell command and exits on failure.""" + use_shell = shell + # On Windows, list commands usually need shell=True to find executables + if platform.system() == "Windows" and isinstance(command, list): + use_shell = True + cmd_str = ' '.join(command) if isinstance(command, list) else command print(f"--> [EXEC]: {cmd_str}") + try: - subprocess.check_call(command, shell=shell, cwd=cwd) + subprocess.check_call(command, shell=use_shell, cwd=cwd) except subprocess.CalledProcessError as e: print(f"Error executing command: {e}") sys.exit(1) -def is_python311_installed(): - """Checks if python3.11 is currently available in the system PATH.""" - # Check standard PATH - if shutil.which("python3.11") is not None: - return True +def get_os_type(): + """Returns 'windows', 'macos', or 'linux'.""" + os_name = platform.system().lower() - # Check common local install location (Source builds often go here) - if os.path.exists("/usr/local/bin/python3.11"): - return True - - return False - -def check_system_compatibility(): - """Ensures we are on a Debian-based system (Debian, Ubuntu, Mint, Kali, etc).""" - if shutil.which("apt-get") is None: - print("\n!!! CRITICAL ERROR !!!") - print("This script relies on 'apt-get'. It works on Debian, Ubuntu, Linux Mint, Kali, Pop!_OS, etc.") - sys.exit(1) + if "darwin" in os_name: + return "macos" + elif "win" in os_name: + return "windows" + return "linux" + +def find_python311(): + """Finds python3.11 executable path cross-platform.""" + candidates = [] + + if get_os_type() == "windows": + candidates = ["py", "python", "python3.11"] + else: + # Check standard PATH first, then explicit locations + candidates = ["python3.11", "/usr/bin/python3.11", "/usr/local/bin/python3.11", "/opt/homebrew/bin/python3.11"] + + for cmd in candidates: + path = shutil.which(cmd) + if path: + try: + # Verify it is actually 3.11 + # We use --version to check the actual output + ver = subprocess.check_output([path, "--version"]).decode().strip() + if "3.11" in ver: + return path + except: + continue + return None + +def install_linux_strategy(): + """Installs Python 3.11 on Linux (Ubuntu PPA or Debian Source).""" + print("\n[Linux] Python 3.11 not found. Determining installation strategy...") + + if not shutil.which("apt-get"): + print("Error: This script requires 'apt-get' (Debian/Ubuntu/Mint/Kali).") + sys.exit(1) -def is_ubuntu(): - """Returns True only if we are confident this is Ubuntu.""" + # 1. Detect Ubuntu (Use PPA for speed) + is_ubuntu = False try: if os.path.exists("/etc/os-release"): with open("/etc/os-release") as f: if "ubuntu" in f.read().lower(): - return True - if shutil.which("lsb_release"): - out = subprocess.check_output(["lsb_release", "-i"]).decode().lower() - if "ubuntu" in out: - return True + is_ubuntu = True except: pass - return False -def install_python_ubuntu_strategy(): - """Strategy A: Ubuntu (Fast PPA)""" - print("\n[Strategy] Detected Ubuntu. Using fast PPA installation...") - run_command("sudo apt-get update", shell=True) - run_command("sudo apt-get install -y software-properties-common git", shell=True) - print("Adding Deadsnakes PPA...") - run_command("sudo add-apt-repository -y ppa:deadsnakes/ppa", shell=True) run_command("sudo apt-get update", shell=True) - run_command("sudo apt-get install -y python3.11 python3.11-venv python3.11-dev", shell=True) -def install_python_debian_strategy(): - """Strategy B: Debian / Universal (Source Build Fallback)""" - print("\n[Strategy] Detected Debian/Other. Using robust compatibility mode...") - - # 1. Install Dependencies - run_command("sudo apt-get update", shell=True) - print("Installing build dependencies...") + if is_ubuntu: + print("Detected Ubuntu. Attempting PPA install...") + try: + run_command("sudo apt-get install -y software-properties-common", shell=True) + run_command("sudo add-apt-repository -y ppa:deadsnakes/ppa", shell=True) + run_command("sudo apt-get update", shell=True) + run_command("sudo apt-get install -y python3.11 python3.11-venv python3.11-dev", shell=True) + return + except Exception as e: + print(f"Ubuntu PPA failed ({e}). Falling back to source build.") + + # 2. Debian/Fallback Strategy (Source Build) + print("Detected Debian/Other. Using Source Build (Robust Method)...") + + # Install Build Dependencies deps = [ - "git", "wget", "build-essential", "zlib1g-dev", "libncurses5-dev", + "wget", "build-essential", "zlib1g-dev", "libncurses5-dev", "libgdbm-dev", "libnss3-dev", "libssl-dev", "libreadline-dev", "libffi-dev", "libsqlite3-dev", "libbz2-dev", "pkg-config" ] run_command(f"sudo apt-get install -y {' '.join(deps)}", shell=True) - - # 2. Try APT first (Debian 12+) - print("Checking system repos for Python 3.11...") - try: - subprocess.check_call("apt-cache show python3.11", shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - print("Python 3.11 found in APT. Installing...") - run_command("sudo apt-get install -y python3.11 python3.11-venv python3.11-dev", shell=True) - return - except subprocess.CalledProcessError: - print("Python 3.11 not in system repos. Proceeding to Source Build.") - - # 3. Source Build (Universal) - print("\n*** STARTING SOURCE BUILD ***") + + # Download & Build tarball = f"Python-{PYTHON_VERSION}.tgz" url = f"https://www.python.org/ftp/python/{PYTHON_VERSION}/{tarball}" - + if not os.path.exists(tarball): run_command(f"wget {url}", shell=True) run_command(f"tar -xf {tarball}", shell=True) src_dir = f"Python-{PYTHON_VERSION}" - # Configure & Make + # Configure & Install + # --enable-optimizations is standard for production python builds run_command("./configure --enable-optimizations", shell=True, cwd=src_dir) nproc = subprocess.check_output("nproc", shell=True).decode().strip() run_command(f"make -j {nproc}", shell=True, cwd=src_dir) - # SAFE INSTALL (altinstall) + # CRITICAL: Use altinstall to avoid overwriting /usr/bin/python3 run_command("sudo make altinstall", shell=True, cwd=src_dir) # Cleanup os.remove(tarball) run_command(f"sudo rm -rf {src_dir}", shell=True) -def setup_project_and_venv(): - print("\n[Project] Setting up VectorDBBench...") +def install_macos_strategy(): + print("\n[macOS] Installing Python 3.11 via Homebrew...") + if shutil.which("brew") is None: + print("Error: Homebrew not found. Please install it from brew.sh") + sys.exit(1) + run_command("brew install python@3.11", shell=True) + +def install_windows_strategy(): + print("\n[Windows] Installing Python 3.11 via Winget/Installer...") + # Try Winget first (standard on Win 10/11) + if shutil.which("winget"): + try: + run_command("winget install -e --id Python.Python.3.11", shell=True) + return + except: + pass + + # Fallback to direct download + installer = "python-3.11.9-amd64.exe" + url = f"https://www.python.org/ftp/python/3.11.9/{installer}" + print(f"Downloading {url}...") + urllib.request.urlretrieve(url, installer) + run_command([installer, "/quiet", "InstallAllUsers=1", "PrependPath=1"]) + os.remove(installer) + +def setup_project(python_exe): + print(f"\n[Project] Setting up repo using found Python: {python_exe}") # 1. Clone if not os.path.exists(REPO_DIR): @@ -119,57 +159,77 @@ def setup_project_and_venv(): os.chdir(REPO_DIR) - # 2. Switch Branch + # 2. Checkout Branch run_command(["git", "fetch", "origin"]) run_command(["git", "checkout", "Endee"]) run_command(["git", "pull", "origin", "Endee"]) - # 3. Locate Python 3.11 - python_bin = "python3.11" - if shutil.which("python3.11") is None: - if os.path.exists("/usr/local/bin/python3.11"): - python_bin = "/usr/local/bin/python3.11" - else: - print("Error: Python 3.11 binary not found after installation attempt.") - sys.exit(1) - - print(f"Using Python binary: {python_bin}") - - # 4. Create Venv + # 3. Create Venv if not os.path.exists("venv"): - run_command([python_bin, "-m", "venv", "venv"]) + print("Creating virtual environment...") + run_command([python_exe, "-m", "venv", "venv"]) else: - print("Virtual environment already exists.") + print("Virtual environment already exists. Skipping creation.") + + # 4. Install Deps + # Check OS *again* here to determine correct PIP path + if get_os_type() == "windows": + venv_pip = os.path.join("venv", "Scripts", "pip.exe") + else: + # MacOS and Linux use this path + venv_pip = os.path.join("venv", "bin", "pip") - # 5. Install Deps - venv_pip = os.path.join("venv", "bin", "pip") + print(f"Installing dependencies using: {venv_pip}") run_command([venv_pip, "install", "--upgrade", "pip"]) run_command([venv_pip, "install", "endee"]) run_command([venv_pip, "install", "-e", "."]) - return os.path.join(os.getcwd(), "venv") + return venv_pip if __name__ == "__main__": - check_system_compatibility() + # 0. Check Git + if shutil.which("git") is None: + print("Error: Git is not installed.") + if get_os_type() == "linux": + run_command("sudo apt-get update && sudo apt-get install -y git", shell=True) + else: + sys.exit(1) - # --- THE FIX IS HERE --- - if is_python311_installed(): + # 1. Check for Existing Python 3.11 + py_path = find_python311() + + if py_path: print("\n" + "="*50) - print("SKIP: Python 3.11 is already installed.") + print(f"FOUND PYTHON 3.11: {py_path}") + print("Skipping OS installation steps.") print("="*50) else: - # Only install if missing - if is_ubuntu(): - install_python_ubuntu_strategy() - else: - install_python_debian_strategy() - # ----------------------- + # Install if missing + os_type = get_os_type() + if os_type == "linux": + install_linux_strategy() + elif os_type == "macos": + install_macos_strategy() + elif os_type == "windows": + install_windows_strategy() + + # Verify installation + py_path = find_python311() + if not py_path: + print("Error: Installation failed or binary not found.") + sys.exit(1) - venv_path = setup_project_and_venv() + # 2. Setup Project + setup_project(py_path) print("\n" + "="*50) print("SETUP SUCCESSFUL!") - print(f"To start: source {os.path.join(venv_path, 'bin', 'activate')}") + print("="*50) + + if get_os_type() == "windows": + print(f"To start: {os.path.join(os.getcwd(), 'venv', 'Scripts', 'activate')}") + else: + print(f"To start: source {os.path.join(os.getcwd(), 'venv', 'bin', 'activate')}") print("="*50) From e554491b6cc45128f882f4fa8105acad82487aec Mon Sep 17 00:00:00 2001 From: Mithun Kumar Date: Wed, 7 Jan 2026 16:05:26 +0530 Subject: [PATCH 14/21] Modified cli implementation --- vectordb_bench/backend/clients/endee/cli.py | 47 ++++++++++++++++++--- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/vectordb_bench/backend/clients/endee/cli.py b/vectordb_bench/backend/clients/endee/cli.py index 970e4ce7a..57c96560c 100644 --- a/vectordb_bench/backend/clients/endee/cli.py +++ b/vectordb_bench/backend/clients/endee/cli.py @@ -78,20 +78,19 @@ def Endee(**parameters): collection_name = f"endee_bench_{uuid.uuid4().hex[:8]}" # Filter out None values before creating config - params_for_vecx = {k: v for k, v in parameters.items() if v is not None} - db_config = EndeeConfig(**params_for_vecx) + params_for_nd = {k: v for k, v in parameters.items() if v is not None} + db_config = EndeeConfig(**params_for_nd) custom_case_config = get_custom_case_config(parameters) # Create task config from vectordb_bench.models import TaskConfig, CaseConfig, CaseType, ConcurrencySearchConfig - # Create an instance of EmptyDBCaseConfig instead of passing None db_case_config = EmptyDBCaseConfig() task = TaskConfig( db=DB.Endee, - db_config=db_config, # Use the EndeeConfig instance directly + db_config=db_config, db_case_config=db_case_config, case_config=CaseConfig( case_id=CaseType[parameters["case_type"]], @@ -99,6 +98,7 @@ def Endee(**parameters): concurrency_search_config=ConcurrencySearchConfig( concurrency_duration=parameters["concurrency_duration"], num_concurrency=[int(s) for s in parameters["num_concurrency"]], + concurrency_timeout=parameters["concurrency_timeout"], # ========== Added this ========== ), custom_case=custom_case_config, ), @@ -107,13 +107,46 @@ def Endee(**parameters): # Use the run method of the benchmark_runner object if not parameters["dry_run"]: - benchmark_runner.run([task]) + # --- TASK LABEL LOGIC START --- + # 1. Generate a unique run ID + run_uuid = uuid.uuid4().hex + + # 2. Pick the base name: Use --task-label if provided, else --index-name + base_name = parameters.get("task_label") + if not base_name: + base_name = parameters.get("index_name", "endee") + + # 3. Combine into the final label: "MyLabel_UUID" + final_label = f"{base_name}_{run_uuid}" + + # 4. Pass the label to the runner to ensure the filename is correct + benchmark_runner.run([task], final_label) + # --- TASK LABEL LOGIC END --- - # Wait for task to complete if needed + # Wait for task to complete import time from vectordb_bench.interface import global_result_future from concurrent.futures import wait time.sleep(5) if global_result_future: - wait([global_result_future]) \ No newline at end of file + wait([global_result_future]) + + # Ensure CLI doesn't close while background processes are active + while benchmark_runner.has_running(): + time.sleep(1) + + + +# # Use the run method of the benchmark_runner object +# if not parameters["dry_run"]: +# benchmark_runner.run([task]) + +# # Wait for task to complete if needed +# import time +# from vectordb_bench.interface import global_result_future +# from concurrent.futures import wait + +# time.sleep(5) +# if global_result_future: +# wait([global_result_future]) \ No newline at end of file From 309020f603e4f059e0967abbbc6da4b1701a1126 Mon Sep 17 00:00:00 2001 From: darshan-endee Date: Sat, 10 Jan 2026 19:25:11 +0530 Subject: [PATCH 15/21] filter_endee --- demo_cli_ommands.txt | 77 +++- vectordb_bench/backend/clients/endee/endee.py | 372 +++++++++++++++--- 2 files changed, 394 insertions(+), 55 deletions(-) diff --git a/demo_cli_ommands.txt b/demo_cli_ommands.txt index a770786bc..8e7e55d7a 100644 --- a/demo_cli_ommands.txt +++ b/demo_cli_ommands.txt @@ -57,7 +57,7 @@ DATASET_LOCAL_DIR="/home/admin/vectordataset" vectordbbench endee \ NUM_PER_BATCH=10000 DATASET_LOCAL_DIR="/home/admin/vectordataset" vectordbbench endee \ --token "TOKEN" \ --region india-west-1 \ - --base-url "http://127.0.0.1:8080/api/v1" \ + --base-url "https://dev.endee.io/api/v1" \ --index-name test_index_name \ --task-label "20260107" \ --m 16 \ @@ -75,3 +75,78 @@ NUM_PER_BATCH=10000 DATASET_LOCAL_DIR="/home/admin/vectordataset" vectordbbench --load \ --search-concurrent \ --search-serial + + + +DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset" vectordbbench endee \ + --token "x6zdfkgq:pd6fnXWS8e2ZkqAUNFBfirYcHqFShXgR:dev" \ + --region india-west-1 \ + --base-url "https://dev.endee.io/api/v1" \ + --index-name test_1M_numfilter_1p \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int8d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.01 \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset" vectordbbench endee \ + --token "x6zdfkgq:pd6fnXWS8e2ZkqAUNFBfirYcHqFShXgR:dev" \ + --region india-west-1 \ + --base-url "https://dev.endee.io/api/v1" \ + --index-name test_1M_numfilter_1p \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int8d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.01 \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + +DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset_label_filter" vectordbbench endee \ + --token "x6zdfkgq:pd6fnXWS8e2ZkqAUNFBfirYcHqFShXgR:dev" \ + --region india-west-1 \ + --base-url "https://dev.endee.io/api/v1" \ + --index-name test_1M_labelfilter \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int8d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.01 \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial diff --git a/vectordb_bench/backend/clients/endee/endee.py b/vectordb_bench/backend/clients/endee/endee.py index a87efbf37..fed841cfb 100644 --- a/vectordb_bench/backend/clients/endee/endee.py +++ b/vectordb_bench/backend/clients/endee/endee.py @@ -1,8 +1,238 @@ +# import logging +# from contextlib import contextmanager + +# from endee import endee + +# from ..api import DBCaseConfig, EmptyDBCaseConfig, IndexType, VectorDB +# from .config import EndeeConfig + +# log = logging.getLogger(__name__) + + +# class Endee(VectorDB): +# def __init__( +# self, +# dim: int, +# db_config: dict, +# db_case_config: DBCaseConfig, +# drop_old: bool = False, +# **kwargs, +# ): +# print(db_config) + +# self.token = db_config.get("token", "") +# self.region = db_config.get("region", "india-west-1") +# self.base_url = db_config.get("base_url") + +# # self.collection_name = db_config.get("collection_name", "") +# self.collection_name = (db_config.get("collection_name") or db_config.get("index_name")) + +# if not self.collection_name: +# import uuid +# self.collection_name = f"endee_bench_{uuid.uuid4().hex[:8]}" + +# self.space_type = db_config.get("space_type", "cosine") +# # self.use_fp16 = db_config.get("use_fp16", False) +# self.precision = db_config.get("precision") +# self.version = db_config.get("version") +# self.M = db_config.get("m") +# self.ef_con = db_config.get("ef_con") +# self.ef_search = db_config.get("ef_search") +# self.nd = endee.Endee(token=self.token) + +# # Dynamically set the URL +# if self.base_url: +# self.nd.set_base_url(self.base_url) +# log.info(f"Targeting server: {self.base_url}") + +# # Using Base url +# # self.nd.base_url = "http://10.128.0.3:8080/api/v1" +# # self.nd.set_base_url("http://10.128.0.3:8080/api/v1") +# # self.nd.set_base_url("http://3.85.217.253:80/api/v1") +# # self.nd.set_base_url("http://10.128.0.5:8080/api/v1") +# # self.nd.set_base_url("http://54.89.169.48:80/api/v1") + + +# # BASE_URL="http://3.85.217.253:80/api/v1" + +# # try: +# # indices = self.nd.list_indexes().get("indices", []) +# # index_names = [index["name"] for index in indices] if indices else [] + +# # if drop_old and self.collection_name in index_names: +# # self._drop_index(self.collection_name) +# # self._create_index(dim) +# # elif self.collection_name not in index_names: +# # self._create_index(dim) +# # except Exception as e: +# # log.error(f"Error connecting to Endee API: {e}") +# # raise + +# try: +# index_name = self.collection_name # assign before use +# indices = self.nd.list_indexes().get("indices", []) +# index_names = [index["name"] for index in indices] if indices else [] +# try: +# self.index = self.nd.get_index(name=index_name) +# log.info(f"Connected to existing Endee index: '{index_name}'") + +# except Exception as fetch_error: +# log.warning(f"Index '{index_name}' not found. Creating new index...") +# try: +# self._create_index(dim) +# self.index = self.nd.get_index(name=index_name) +# log.info(f"Successfully created and connected to index: '{index_name}'") + +# except Exception as create_error: +# # If error is "index already exists", just get the index again +# if "already exists" in str(create_error).lower() or "conflict" in str(create_error).lower(): +# log.warning(f"Index '{index_name}' already exists despite previous error. Fetching it again.") +# self.index = self.nd.get_index(name=index_name) +# else: +# log.error(f"Failed to create Endee index: {create_error}") +# raise +# except Exception as e: +# log.error(f"Error accessing or creating Endee index '{index_name}': {e}") +# raise + + + +# def _create_index(self, dim): +# try: +# resp = self.nd.create_index( +# name=self.collection_name, +# dimension=dim, +# space_type=self.space_type, +# # use_fp16=self.use_fp16, +# precision=self.precision, +# version=self.version, +# M=self.M, +# ef_con=self.ef_con +# ) +# log.info(f"Created new Endee index: {resp}") +# except Exception as e: +# log.error(f"Failed to create Endee index: {e}") +# raise + +# def _drop_index(self, collection_name): +# try: +# res = self.nd.delete_index(collection_name) +# log.info(res) +# except Exception as e: +# log.error(f"Failed to drop Endee index: {e}") +# raise + +# @classmethod +# def config_cls(cls) -> type[EndeeConfig]: +# return EndeeConfig + +# @classmethod +# def case_config_cls(cls, index_type: IndexType | None = None) -> type[DBCaseConfig]: +# return EmptyDBCaseConfig + +# @contextmanager +# def init(self): +# try: +# # log.info(f"Token: {self.token}") +# nd = endee.Endee(token=self.token) +# # Uncomment below to use base_url +# # nd.base_url = "http://10.128.0.3:8080/api/v1" +# # nd.set_base_url("http://10.128.0.3:8080/api/v1") +# # nd.set_base_url("http://3.85.217.253:80/api/v1") +# # nd.set_base_url("http://10.128.0.5:8080/api/v1") +# # nd.set_base_url("http://54.89.169.48:80/api/v1") + +# if self.base_url: +# nd.set_base_url(self.base_url) +# self.nd = nd + +# self.index = nd.get_index(name=self.collection_name) +# yield +# except Exception as e: +# if hasattr(e, 'response') and e.response is not None: +# log.error(f"HTTP Status: {e.response.status_code}, Body: {e.response.text}") +# log.error(f"Error initializing Endee client: {e}") +# # log.error("Error initializing Endee client", exc_info=True) +# raise +# finally: +# pass + +# def optimize(self, data_size: int | None = None): +# pass + +# def insert_embeddings( +# self, +# embeddings: list[list[float]], +# metadata: list[int], +# **kwargs, +# ) -> (int, Exception): # type: ignore +# assert len(embeddings) == len(metadata) +# insert_count = 0 +# try: +# batch_vectors = [ +# { +# "id": str(metadata[i]), +# "vector": embeddings[i], +# "meta": {"id": str(metadata[i])} +# } +# for i in range(len(embeddings)) +# ] +# self.index.upsert(batch_vectors) +# insert_count = len(batch_vectors) + +# except Exception as e: +# return (insert_count, e) + +# return (len(embeddings), None) + +# def search_embedding( +# self, +# query: list[float], +# k: int = 30, +# filters: dict | None = None, +# **kwargs, +# ) -> list[int]: +# try: +# filter_expr = None +# if filters and "id" in filters: +# filter_expr = {"id": filters["id"]} + +# results = self.index.query( +# vector=query, +# top_k=k, +# filter=filter_expr, +# ef=self.ef_search, +# include_vectors=False +# ) + +# return [int(result["id"]) for result in results] + +# except Exception as e: +# log.warning(f"Error querying Endee index: {e}") +# raise + +# def describe_index(self) -> dict: +# """Get information about the current index.""" +# try: +# all_indices = self.nd.list_indexes().get("indices", []) + +# for idx in all_indices: +# if idx.get("name") == self.collection_name: +# return idx + +# return {} +# except Exception as e: +# log.warning(f"Error describing Endee index: {e}") +# return {} + import logging from contextlib import contextmanager +from collections.abc import Iterable from endee import endee +from vectordb_bench.backend.filter import Filter, FilterOp + from ..api import DBCaseConfig, EmptyDBCaseConfig, IndexType, VectorDB from .config import EndeeConfig @@ -10,12 +240,20 @@ class Endee(VectorDB): + # Add supported filter types like Milvus + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + FilterOp.StrEqual, + ] + def __init__( self, dim: int, db_config: dict, db_case_config: DBCaseConfig, drop_old: bool = False, + with_scalar_labels: bool = False, **kwargs, ): print(db_config) @@ -24,7 +262,6 @@ def __init__( self.region = db_config.get("region", "india-west-1") self.base_url = db_config.get("base_url") - # self.collection_name = db_config.get("collection_name", "") self.collection_name = (db_config.get("collection_name") or db_config.get("index_name")) if not self.collection_name: @@ -32,12 +269,20 @@ def __init__( self.collection_name = f"endee_bench_{uuid.uuid4().hex[:8]}" self.space_type = db_config.get("space_type", "cosine") - # self.use_fp16 = db_config.get("use_fp16", False) self.precision = db_config.get("precision") self.version = db_config.get("version") self.M = db_config.get("m") self.ef_con = db_config.get("ef_con") self.ef_search = db_config.get("ef_search") + self.with_scalar_labels = with_scalar_labels + + # Initialize filter expression (similar to Milvus) + self.filter_expr = None + + # Field names to match Milvus convention + self._scalar_id_field = "id" + self._scalar_label_field = "label" + self.nd = endee.Endee(token=self.token) # Dynamically set the URL @@ -45,31 +290,8 @@ def __init__( self.nd.set_base_url(self.base_url) log.info(f"Targeting server: {self.base_url}") - # Using Base url - # self.nd.base_url = "http://10.128.0.3:8080/api/v1" - # self.nd.set_base_url("http://10.128.0.3:8080/api/v1") - # self.nd.set_base_url("http://3.85.217.253:80/api/v1") - # self.nd.set_base_url("http://10.128.0.5:8080/api/v1") - # self.nd.set_base_url("http://54.89.169.48:80/api/v1") - - - # BASE_URL="http://3.85.217.253:80/api/v1" - - # try: - # indices = self.nd.list_indexes().get("indices", []) - # index_names = [index["name"] for index in indices] if indices else [] - - # if drop_old and self.collection_name in index_names: - # self._drop_index(self.collection_name) - # self._create_index(dim) - # elif self.collection_name not in index_names: - # self._create_index(dim) - # except Exception as e: - # log.error(f"Error connecting to Endee API: {e}") - # raise - try: - index_name = self.collection_name # assign before use + index_name = self.collection_name indices = self.nd.list_indexes().get("indices", []) index_names = [index["name"] for index in indices] if indices else [] try: @@ -84,7 +306,6 @@ def __init__( log.info(f"Successfully created and connected to index: '{index_name}'") except Exception as create_error: - # If error is "index already exists", just get the index again if "already exists" in str(create_error).lower() or "conflict" in str(create_error).lower(): log.warning(f"Index '{index_name}' already exists despite previous error. Fetching it again.") self.index = self.nd.get_index(name=index_name) @@ -95,15 +316,12 @@ def __init__( log.error(f"Error accessing or creating Endee index '{index_name}': {e}") raise - - def _create_index(self, dim): try: resp = self.nd.create_index( name=self.collection_name, dimension=dim, space_type=self.space_type, - # use_fp16=self.use_fp16, precision=self.precision, version=self.version, M=self.M, @@ -133,14 +351,7 @@ def case_config_cls(cls, index_type: IndexType | None = None) -> type[DBCaseConf @contextmanager def init(self): try: - # log.info(f"Token: {self.token}") nd = endee.Endee(token=self.token) - # Uncomment below to use base_url - # nd.base_url = "http://10.128.0.3:8080/api/v1" - # nd.set_base_url("http://10.128.0.3:8080/api/v1") - # nd.set_base_url("http://3.85.217.253:80/api/v1") - # nd.set_base_url("http://10.128.0.5:8080/api/v1") - # nd.set_base_url("http://54.89.169.48:80/api/v1") if self.base_url: nd.set_base_url(self.base_url) @@ -152,7 +363,6 @@ def init(self): if hasattr(e, 'response') and e.response is not None: log.error(f"HTTP Status: {e.response.status_code}, Body: {e.response.text}") log.error(f"Error initializing Endee client: {e}") - # log.error("Error initializing Endee client", exc_info=True) raise finally: pass @@ -160,50 +370,104 @@ def init(self): def optimize(self, data_size: int | None = None): pass + # def prepare_filter(self, filters: Filter): + # """ + # Prepare filter expression for Endee based on filter type. + # Similar to Milvus's prepare_filter method. + # """ + # if filters.type == FilterOp.NonFilter: + # self.filter_expr = None + # elif filters.type == FilterOp.NumGE: + # # For numeric >= filter, use $range operator + # # filters.int_value is the minimum value + # self.filter_expr = [{self._scalar_id_field: {"$range": [filters.int_value, float('inf')]}}] + # elif filters.type == FilterOp.StrEqual: + # # For string equality filter, use $eq operator + # self.filter_expr = [{self._scalar_label_field: {"$eq": filters.label_value}}] + # else: + # msg = f"Not support Filter for Endee - {filters}" + # raise ValueError(msg) + def prepare_filter(self, filters: Filter): + if filters.type == FilterOp.NonFilter: + self.filter_expr = None + + elif filters.type == FilterOp.NumGE: + # Endee supports $range, NOT $gte + # Dataset size = 1 million → finite upper bound + self.filter_expr = [ + {self._scalar_id_field: {"$range": [filters.int_value, 1_000_000]}} + ] + + elif filters.type == FilterOp.StrEqual: + self.filter_expr = [ + {self._scalar_label_field: {"$eq": filters.label_value}} + ] + + else: + raise ValueError(f"Not support Filter for Endee - {filters}") + + def insert_embeddings( self, - embeddings: list[list[float]], + embeddings: Iterable[list[float]], metadata: list[int], + labels_data: list[str] | None = None, **kwargs, - ) -> (int, Exception): # type: ignore + ) -> tuple[int, Exception]: + """ + Insert embeddings with filter metadata. + Modified to include filter fields like Milvus does. + """ assert len(embeddings) == len(metadata) insert_count = 0 try: - batch_vectors = [ - { + batch_vectors = [] + for i in range(len(embeddings)): + vector_data = { "id": str(metadata[i]), "vector": embeddings[i], - "meta": {"id": str(metadata[i])} + "meta": {"id": metadata[i]}, # Store id in meta for reference + "filter": { + self._scalar_id_field: metadata[i] # Store id for numeric filtering + } } - for i in range(len(embeddings)) - ] + + # Add label field if using scalar labels + if self.with_scalar_labels and labels_data is not None: + vector_data["filter"][self._scalar_label_field] = labels_data[i] + + batch_vectors.append(vector_data) + self.index.upsert(batch_vectors) insert_count = len(batch_vectors) except Exception as e: - return (insert_count, e) + log.error(f"Failed to insert data: {e}") + return insert_count, e return (len(embeddings), None) def search_embedding( self, query: list[float], - k: int = 30, - filters: dict | None = None, + k: int = 100, + timeout: int | None = None, **kwargs, ) -> list[int]: + """ + Perform a search with filter expression. + Modified to use prepared filter like Milvus does. + """ try: - filter_expr = None - if filters and "id" in filters: - filter_expr = {"id": filters["id"]} - + # print("Filter expression:",self.filter_expr) results = self.index.query( vector=query, top_k=k, - filter=filter_expr, + filter=self.filter_expr, # Use prepared filter expression ef=self.ef_search, include_vectors=False ) + # print("Results:",results) return [int(result["id"]) for result in results] @@ -223,4 +487,4 @@ def describe_index(self) -> dict: return {} except Exception as e: log.warning(f"Error describing Endee index: {e}") - return {} + return {} \ No newline at end of file From d2e895d967e6f3908c2c0e9797a8d2586ba9ab2a Mon Sep 17 00:00:00 2001 From: darshan-endee Date: Thu, 22 Jan 2026 12:54:51 +0530 Subject: [PATCH 16/21] changed --- .gitignore | 7 ++- demo_cli_ommands.txt | 51 ++++++++++++++++++- vectordb_bench/backend/clients/endee/endee.py | 1 + 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index aa3a72f44..fe2e2988a 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,9 @@ venv/ .venv/ .idea/ results/ -logs/ \ No newline at end of file +logs/ +neighbors_labels_label_1p.csv +vectordataset/ +vectordataset_label_filter/ +test_query.csv +test_ids.txt \ No newline at end of file diff --git a/demo_cli_ommands.txt b/demo_cli_ommands.txt index 8e7e55d7a..7ecf581ac 100644 --- a/demo_cli_ommands.txt +++ b/demo_cli_ommands.txt @@ -116,7 +116,7 @@ DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset" vectord --version 1 \ --case-type NewIntFilterPerformanceCase \ --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ - --filter-rate 0.01 \ + --filter-rate 0.99 \ --k 30 \ --num-concurrency "8" \ --concurrency-duration 30 \ @@ -150,3 +150,52 @@ DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset_label_fi --load \ --search-concurrent \ --search-serial + + + DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset" vectordbbench endee \ + --token "x6zdfkgq:pd6fnXWS8e2ZkqAUNFBfirYcHqFShXgR:dev" \ + --region india-west-1 \ + --base-url "https://dev.endee.io/api/v1" \ + --index-name test_1M_numfilter_1p \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int8d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.01 \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset_label_filter" vectordbbench endee \ + --token "x6zdfkgq:pd6fnXWS8e2ZkqAUNFBfirYcHqFShXgR:dev" \ + --region india-west-1 \ + --base-url "https://dev.endee.io/api/v1" \ + --index-name test_1M_labelfilter \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int8d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.01 \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial \ No newline at end of file diff --git a/vectordb_bench/backend/clients/endee/endee.py b/vectordb_bench/backend/clients/endee/endee.py index fed841cfb..bedec1a88 100644 --- a/vectordb_bench/backend/clients/endee/endee.py +++ b/vectordb_bench/backend/clients/endee/endee.py @@ -460,6 +460,7 @@ def search_embedding( """ try: # print("Filter expression:",self.filter_expr) + # print("Query:", query) results = self.index.query( vector=query, top_k=k, From 484b8be84109199ece56623cfbcf183d4d5bf50e Mon Sep 17 00:00:00 2001 From: darshan-endee Date: Mon, 23 Feb 2026 06:26:28 +0000 Subject: [PATCH 17/21] filter_parameters_addition_stability_test_function_insert_embeddings --- .gitignore | 6 +- vectordb_bench/backend/clients/endee/cli.py | 9 +- .../backend/clients/endee/config.py | 5 + vectordb_bench/backend/clients/endee/endee.py | 116 ++++++++++++++++-- 4 files changed, 123 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index fe2e2988a..c0f081862 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,8 @@ neighbors_labels_label_1p.csv vectordataset/ vectordataset_label_filter/ test_query.csv -test_ids.txt \ No newline at end of file +test_ids.txt +debug_logs/ +ground_truth_ids.csv +matched_metadata.txt +vectordataset_label/ \ No newline at end of file diff --git a/vectordb_bench/backend/clients/endee/cli.py b/vectordb_bench/backend/clients/endee/cli.py index 57c96560c..e51f605bf 100644 --- a/vectordb_bench/backend/clients/endee/cli.py +++ b/vectordb_bench/backend/clients/endee/cli.py @@ -59,7 +59,14 @@ class EndeeTypedDict(CommonTypedDict): str, click.option("--index-name", type=str, required=True, help="Endee index name (will use a random name if not provided)", show_default=True) ] - + prefilter_cardinality_threshold: Annotated[ + int, + click.option("--prefilter-cardinality-threshold", type=int, default=None, help="Switch to brute-force prefiltering when filter matches ≤N vectors. Range: 1,000–1,000,000 (default: 10,000).", show_default=True) + ] + filter_boost_percentage: Annotated[ + int, + click.option("--filter-boost-percentage", type=int, default=None, help="Expand HNSW candidate pool by N%% when a filter is active to compensate for filtered-out results. Range: 0–100 (default: 0).", show_default=True) + ] @click.command() diff --git a/vectordb_bench/backend/clients/endee/config.py b/vectordb_bench/backend/clients/endee/config.py index aa4bf8c5d..4bb6b50a3 100644 --- a/vectordb_bench/backend/clients/endee/config.py +++ b/vectordb_bench/backend/clients/endee/config.py @@ -16,6 +16,9 @@ class EndeeConfig(DBConfig): ef_search: Optional[int] = 128 # collection_name: str index_name: str + prefilter_cardinality_threshold: Optional[int] = 10000 + filter_boost_percentage: Optional[int] = 0 + def to_dict(self) -> dict: return { @@ -31,4 +34,6 @@ def to_dict(self) -> dict: "ef_search": self.ef_search, # "collection_name": self.collection_name, "index_name": self.index_name, + "prefilter_cardinality_threshold": self.prefilter_cardinality_threshold, + "filter_boost_percentage": self.filter_boost_percentage, } \ No newline at end of file diff --git a/vectordb_bench/backend/clients/endee/endee.py b/vectordb_bench/backend/clients/endee/endee.py index bedec1a88..dae5d1ed3 100644 --- a/vectordb_bench/backend/clients/endee/endee.py +++ b/vectordb_bench/backend/clients/endee/endee.py @@ -225,6 +225,7 @@ # log.warning(f"Error describing Endee index: {e}") # return {} +import time import logging from contextlib import contextmanager from collections.abc import Iterable @@ -274,6 +275,8 @@ def __init__( self.M = db_config.get("m") self.ef_con = db_config.get("ef_con") self.ef_search = db_config.get("ef_search") + self.prefilter_cardinality_threshold = db_config.get("prefilter_cardinality_threshold") + self.filter_boost_percentage = db_config.get("filter_boost_percentage") self.with_scalar_labels = with_scalar_labels # Initialize filter expression (similar to Milvus) @@ -407,6 +410,91 @@ def prepare_filter(self, filters: Filter): raise ValueError(f"Not support Filter for Endee - {filters}") + # def insert_embeddings( + # self, + # embeddings: Iterable[list[float]], + # metadata: list[int], + # labels_data: list[str] | None = None, + # **kwargs, + # ) -> tuple[int, Exception]: + # """ + # Insert embeddings with filter metadata. + # Modified to include filter fields like Milvus does. + # """ + # assert len(embeddings) == len(metadata) + # insert_count = 0 + # try: + # batch_vectors = [] + # for i in range(len(embeddings)): + # vector_data = { + # "id": str(metadata[i]), + # "vector": embeddings[i], + # "meta": {"id": metadata[i]}, # Store id in meta for reference + # "filter": { + # self._scalar_id_field: metadata[i] # Store id for numeric filtering + # } + # } + + # # Add label field if using scalar labels + # if self.with_scalar_labels and labels_data is not None: + # vector_data["filter"][self._scalar_label_field] = labels_data[i] + + # batch_vectors.append(vector_data) + + # self.index.upsert(batch_vectors) + # insert_count = len(batch_vectors) + + # except Exception as e: + # log.error(f"Failed to insert data: {e}") + # return insert_count, e + + # return (len(embeddings), None) + + + # def insert_embeddings( + # self, + # embeddings: Iterable[list[float]], + # metadata: list[int], + # labels_data: list[str] | None = None, + # **kwargs, + # ) -> tuple[int, Exception]: + # """ + # Insert embeddings with filter metadata. + # Modified to include filter fields like Milvus does. + # """ + # assert len(embeddings) == len(metadata) + # insert_count = 0 + # try: + # batch_vectors = [] + # for i in range(len(embeddings)): + # if labels_data is None or labels_data[i] != "label_1p": + # continue + # vector_data = { + # "id": str(metadata[i]), + # "vector": embeddings[i], + # "meta": {"id": metadata[i]}, # Store id in meta for reference + # "filter": { + # self._scalar_id_field: metadata[i] # Store id for numeric filtering + # } + # } + + # # Add label field if using scalar labels + # if self.with_scalar_labels and labels_data is not None: + # vector_data["filter"][self._scalar_label_field] = labels_data[i] + + # batch_vectors.append(vector_data) + + # if batch_vectors != []: + # self.index.upsert(batch_vectors) + # insert_count = len(batch_vectors) + + # except Exception as e: + # log.error(f"Failed to insert data: {e}") + # return insert_count, e + + # return (len(embeddings), None) + + def insert_embeddings( self, embeddings: Iterable[list[float]], @@ -414,32 +502,36 @@ def insert_embeddings( labels_data: list[str] | None = None, **kwargs, ) -> tuple[int, Exception]: - """ - Insert embeddings with filter metadata. - Modified to include filter fields like Milvus does. - """ assert len(embeddings) == len(metadata) insert_count = 0 try: batch_vectors = [] for i in range(len(embeddings)): + if metadata[i] > 9999 or metadata[i] < 0: + continue vector_data = { "id": str(metadata[i]), "vector": embeddings[i], - "meta": {"id": metadata[i]}, # Store id in meta for reference + "meta": {"id": metadata[i]}, "filter": { - self._scalar_id_field: metadata[i] # Store id for numeric filtering + self._scalar_id_field: metadata[i] } } - # Add label field if using scalar labels if self.with_scalar_labels and labels_data is not None: vector_data["filter"][self._scalar_label_field] = labels_data[i] batch_vectors.append(vector_data) - - self.index.upsert(batch_vectors) - insert_count = len(batch_vectors) + + # # Log matched metadata IDs to file + # with open("matched_metadata.txt", "a") as f: + # for v in batch_vectors: + # f.write(v["id"] + "\n") + + if batch_vectors != []: + self.index.upsert(batch_vectors) + insert_count = len(batch_vectors) + # time.sleep(20) except Exception as e: log.error(f"Failed to insert data: {e}") @@ -466,7 +558,9 @@ def search_embedding( top_k=k, filter=self.filter_expr, # Use prepared filter expression ef=self.ef_search, - include_vectors=False + include_vectors=False, + prefilter_cardinality_threshold=self.prefilter_cardinality_threshold, + filter_boost_percentage=self.filter_boost_percentage ) # print("Results:",results) From ec026bac4d36733fbe5d643a5d3ff05322bfa24f Mon Sep 17 00:00:00 2001 From: darshan-endee Date: Fri, 27 Feb 2026 08:57:37 +0000 Subject: [PATCH 18/21] scripts addition --- .gitignore | 3 +- demo_cli_ommands.txt | 911 +- intfilterscript.ipynb | 8347 +++++++++++++++++ script.ipynb | 329 + vectordb_bench/backend/clients/endee/endee.py | 137 +- 5 files changed, 9648 insertions(+), 79 deletions(-) create mode 100644 intfilterscript.ipynb create mode 100644 script.ipynb diff --git a/.gitignore b/.gitignore index c0f081862..727820272 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,5 @@ test_ids.txt debug_logs/ ground_truth_ids.csv matched_metadata.txt -vectordataset_label/ \ No newline at end of file +vectordataset_label/ +venv2/ \ No newline at end of file diff --git a/demo_cli_ommands.txt b/demo_cli_ommands.txt index 7ecf581ac..17e427063 100644 --- a/demo_cli_ommands.txt +++ b/demo_cli_ommands.txt @@ -78,11 +78,11 @@ NUM_PER_BATCH=10000 DATASET_LOCAL_DIR="/home/admin/vectordataset" vectordbbench -DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset" vectordbbench endee \ - --token "x6zdfkgq:pd6fnXWS8e2ZkqAUNFBfirYcHqFShXgR:dev" \ +DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ --region india-west-1 \ - --base-url "https://dev.endee.io/api/v1" \ - --index-name test_1M_numfilter_1p \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_numfilter_vnew \ --task-label "20260107" \ --m 16 \ --ef-con 128 \ @@ -92,7 +92,7 @@ DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset" vectord --version 1 \ --case-type NewIntFilterPerformanceCase \ --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ - --filter-rate 0.01 \ + --filter-rate 0.99 \ --k 30 \ --num-concurrency "8" \ --concurrency-duration 30 \ @@ -102,11 +102,11 @@ DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset" vectord --search-concurrent \ --search-serial - DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset" vectordbbench endee \ - --token "x6zdfkgq:pd6fnXWS8e2ZkqAUNFBfirYcHqFShXgR:dev" \ + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ --region india-west-1 \ - --base-url "https://dev.endee.io/api/v1" \ - --index-name test_1M_numfilter_1p \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_numfilter_vnew \ --task-label "20260107" \ --m 16 \ --ef-con 128 \ @@ -116,7 +116,7 @@ DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset" vectord --version 1 \ --case-type NewIntFilterPerformanceCase \ --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ - --filter-rate 0.99 \ + --filter-rate 0.01 \ --k 30 \ --num-concurrency "8" \ --concurrency-duration 30 \ @@ -127,11 +127,11 @@ DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset" vectord --search-serial -DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset_label_filter" vectordbbench endee \ - --token "x6zdfkgq:pd6fnXWS8e2ZkqAUNFBfirYcHqFShXgR:dev" \ +DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ --region india-west-1 \ - --base-url "https://dev.endee.io/api/v1" \ - --index-name test_1M_labelfilter \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_labelfilter_vnew \ --task-label "20260107" \ --m 16 \ --ef-con 128 \ @@ -141,7 +141,7 @@ DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset_label_fi --version 1 \ --case-type LabelFilterPerformanceCase \ --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ - --label-percentage 0.01 \ + --label-percentage 0.50 \ --k 30 \ --num-concurrency "8" \ --concurrency-duration 30 \ @@ -176,11 +176,11 @@ DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset_label_fi --search-concurrent \ --search-serial - DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset_label_filter" vectordbbench endee \ - --token "x6zdfkgq:pd6fnXWS8e2ZkqAUNFBfirYcHqFShXgR:dev" \ + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ --region india-west-1 \ - --base-url "https://dev.endee.io/api/v1" \ - --index-name test_1M_labelfilter \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_labelfilter_vnew \ --task-label "20260107" \ --m 16 \ --ef-con 128 \ @@ -190,11 +190,884 @@ DATASET_LOCAL_DIR="/Users/darshan/Documents/VectorDBBench/vectordataset_label_fi --version 1 \ --case-type LabelFilterPerformanceCase \ --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.001 \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_numfilter_int16 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.80 \ + --k 100 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_numfilter_int16 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.01 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_labelfilter_int16 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.50 \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_labelfilter_int16 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.001 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_numfilter_int16_graphbackfill \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.99 \ + --k 100 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_numfilter_int16_graphbackfill \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.80 \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_labelfilter_int16_graphbackfill \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.02 \ + --k 100 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_labelfilter_int16_graphbackfill \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.001 \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_numfilter_int16_vector_cache \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.99 \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_numfilter_int16_vector_cache \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.99 \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_labelfilter_int16_vector_cache \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ --label-percentage 0.01 \ --k 30 \ --num-concurrency "8" \ --concurrency-duration 30 \ --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_labelfilter_int16_vector_cache \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.001 \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + + DATASET_LOCAL_DIR="/home/debian/VectorDBBench/vectordataset_normal" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_normal_latest_master \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type Performance768D1M \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/VectorDBBench/vectordataset_normal" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_labelfilter_int16_latest_master \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type Performance768D1M \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/VectorDBBench/vectordataset_normal" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_normal_latest_master \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type Performance768D1M \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "q74ehlt6:2Imt8A5iBRz3XQ4mjdcd1Z5R081GgoYq:dev" \ + --region india-west-1 \ + --base-url "https://dev.endee.io/api/v1" \ + --index-name test_1M_numfilter_int16_dev \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.99 \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "q74ehlt6:2Imt8A5iBRz3XQ4mjdcd1Z5R081GgoYq:dev" \ + --region india-west-1 \ + --base-url "https://dev.endee.io/api/v1" \ + --index-name test_1M_numfilter_int16_dev \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.01 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "q74ehlt6:2Imt8A5iBRz3XQ4mjdcd1Z5R081GgoYq:dev" \ + --region india-west-1 \ + --base-url "https://dev.endee.io/api/v1" \ + --index-name test_1M_labelfilter_int16_dev \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.01 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "q74ehlt6:2Imt8A5iBRz3XQ4mjdcd1Z5R081GgoYq:dev" \ + --region india-west-1 \ + --base-url "https://dev.endee.io/api/v1" \ + --index-name test_1M_labelfilter_int16_dev \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.002 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_labelfilter_int16_latest_master \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --prefilter-cardinality-threshold 10000 \ + --filter-boost-percentage 0 \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.01 \ + --k 30 \ + --num-concurrency "8" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_numfilter_int16_latest_master \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.99 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_numfilter_int16_latest_master \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --prefilter-cardinality-threshold 10000 \ + --filter-boost-percentage 0 \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.01 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_labelfilter_int16_latest_master \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --prefilter-cardinality-threshold 10000 \ + --filter-boost-percentage 100 \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.001 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_numfilter_latest_master_stability_bmw \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.99 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_numfilter_latest_master_stability_bmw \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.99 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_dummy \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.99 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_labelfilter_latest_master_stability2 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.02 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_labelfilter_latest_master_stability2 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.001 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset_label" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_labelfilter_latest_master_stability_bmw \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type LabelFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --label-percentage 0.02 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_numfilter_latest_master_stability_vaib2 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.99 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_numfilter_latest_master_stability_vaib2 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.01 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_vaib_reupsertcheck \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.50 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial + + test_1M_vaib_reupsertcheck_backup + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_vaib_reupsertcheck \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.99 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_vaib_reupsertcheck_new1 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.99 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --skip-drop-old \ + --skip-load \ + --search-concurrent \ + --search-serial + + test_1M_vaib_reupsertcheck_new1 + + + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_vaib_reupsertcheck_new1 \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.01 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ --skip-drop-old \ --skip-load \ --search-concurrent \ diff --git a/intfilterscript.ipynb b/intfilterscript.ipynb new file mode 100644 index 000000000..c4e00cf8f --- /dev/null +++ b/intfilterscript.ipynb @@ -0,0 +1,8347 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pip install pandas pyarrow" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pip install endee" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Name: endee\n", + "Version: 0.1.13\n", + "Summary: Endee is the Next-Generation Vector Database for Scalable, High-Performance AI\n", + "Home-page: https://endee.io\n", + "Author: Endee Labs\n", + "Author-email: dev@endee.io\n", + "License: \n", + "Location: /home/debian/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages\n", + "Requires: httpx, msgpack, numpy, orjson, pydantic, requests\n", + "Required-by: \n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "pip show endee" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "base_path = \"/home/debian/latest_VDB/VectorDBBench/vectordataset\"\n", + "dataset_folder = \"cohere/cohere_medium_1m\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " id emb\n", + "0 322406 [0.19600096, -0.5270862, -0.29519123, 0.429556...\n", + "1 337610 [0.32829463, -0.055560444, -0.06252914, -0.101...\n", + "2 714728 [-0.054155815, 0.009554057, 0.24696879, -0.142...\n", + "3 354737 [0.2501778, 0.017853737, -0.43795395, 0.522256...\n", + "4 290979 [0.3444257, -0.62243223, -0.20940691, -0.08620...\n" + ] + } + ], + "source": [ + "#For checking parquet file contents\n", + "import pyarrow.parquet as pq\n", + "import os\n", + "\n", + "file_name = \"shuffle_train.parquet\" #input\n", + "\n", + "# Build full path\n", + "file_path = os.path.join(base_path, dataset_folder, file_name)\n", + "\n", + "\n", + "parquet_file = pq.ParquetFile(file_path)\n", + "\n", + "# Read only first batch of rows\n", + "first_batch = next(parquet_file.iter_batches(batch_size=5))\n", + "preview = first_batch.to_pandas()\n", + "\n", + "for col in preview.columns:\n", + " if preview[col].dtype == object and isinstance(preview[col].iloc[0], list):\n", + " preview[col] = preview[col].apply(lambda x: x[:5] if x is not None else x)\n", + "\n", + "print(preview)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from endee import Endee\n", + "client = Endee(token=\"localtest\")\n", + "client.set_base_url(\"http://148.113.54.173:8080/api/v1\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "test_1M_labelfilter_int16\n", + "1000000\n", + "\t\n", + "test_1M_labelfilter_int16_latest_master\n", + "1000000\n", + "\t\n", + "test_1M_labelfilter_latest_master_stability1\n", + "1000000\n", + "\t\n", + "test_1M_labelfilter_latest_master_stability2\n", + "990000\n", + "\t\n", + "test_1M_normal\n", + "1000000\n", + "\t\n", + "test_1M_normal_latest_master\n", + "1000000\n", + "\t\n", + "test_1M_numfilter_int16\n", + "1000000\n", + "\t\n", + "test_1M_numfilter_int16_latest_master\n", + "1000000\n", + "\t\n", + "test_1M_numfilter_latest_master_stability1\n", + "1000000\n", + "\t\n", + "test_1M_numfilter_latest_master_stability2\n", + "1000000\n", + "\t\n", + "test_1M_numfilter_latest_master_stability3\n", + "990000\n", + "\t\n", + "test_1M_numfilter_latest_master_stability_bmw\n", + "1000000\n", + "\t\n", + "test_1M_numfilter_latest_master_stability_vaib\n", + "1000000\n", + "\t\n", + "test_1M_numfilter_latest_master_stability_vaib2\n", + "200000\n", + "\t\n", + "test_1M_vaib_reupsertcheck\n", + "300000\n", + "\t\n", + "test_1M_vaib_reupsertcheck_new1\n", + "262392\n", + "\t\n" + ] + } + ], + "source": [ + "#For checking indexes\n", + "for obj in client.list_indexes().get('indexes'):\n", + " print(obj['name'])\n", + " print(obj['total_elements'])\n", + " print('\\t')" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "#give the index name\n", + "index_name = \"test_1M_vaib_reupsertcheck_new1\"\n", + "index = client.get_index(index_name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'name': 'test_1M_vaib_reupsertcheck_new1',\n", + " 'space_type': 'cosine',\n", + " 'dimension': 768,\n", + " 'sparse_dim': 0,\n", + " 'is_hybrid': False,\n", + " 'count': 262392,\n", + " 'precision': 'int16',\n", + " 'M': 16}" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "index.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Finding all query IDs with recall < 1.0 ...\n", + "\n", + "Query ID: 0\n", + "Returned IDs: 992169,992979,997002,993717,997637,997918,997822,993288,999453,992790,995401,994922,993289,994819,990399,993738,995994,998530,996438,991266,993032,999587,994334,991150,994964,994409,995701,995708,994804,999450\n", + "Ground Truth: 995888,992169,992979,997002,993717,991151,997637,997918,997822,993288,992933,999453,992790,995401,994922,993289,997852,994819,990399,993478,993738,995994,996777,998530,996438,991266,993412,990337,993032,999587\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 1\n", + "Returned IDs: 997420,995382,990103,993165,996814,996219,996878,992878,998104,994051,994199,990678,995438,992193,994119,997997,992780,993088,997331,992731,996811,998691,999191,991031,995783,999244,993324,996121,995562,993929\n", + "Ground Truth: 997420,995382,990103,993165,996814,996219,996878,992878,998104,994051,994199,990678,996062,995438,993727,992193,994119,997997,999441,996830,992780,993088,997331,990284,992731,997602,997098,996811,998691,999191\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 2\n", + "Returned IDs: 996312,991260,995701,992330,996592,993492,997913,992546,997976,990231,990949,994100,996881,992692,992969,993853,996267,994778,991045,994757,992931,990957,992961,990535,995793,997371,990588,996075,997771,995094\n", + "Ground Truth: 996312,991260,995701,992330,996592,993492,997449,997913,992546,997976,990231,990936,993517,990949,995014,994100,996881,992692,992969,993853,993005,996267,994778,991045,994757,992931,990957,999647,992961,990109\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 3\n", + "Returned IDs: 991304,990167,999555,992444,990627,995610,999994,999033,990194,990437,990434,996209,991798,997084,999131,994355,997086,993076,991667,992892,992517,990394,998335,997798,999280,994564,993480,992628,994836,992778\n", + "Ground Truth: 991304,990167,999555,992444,990627,995610,997415,999994,999033,990194,990437,990434,996209,991798,998279,997084,999131,994355,997086,993076,991667,992892,992517,990394,998335,994725,997798,999280,994564,993480\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 4\n", + "Returned IDs: 993262,991098,996599,992247,997534,990818,990802,992701,995626,999466,991236,997442,998303,991037,996216,997897,994310,995516,997825,999435,992573,990630,994098,997334,991212,993124,997165,994247,991148,993204\n", + "Ground Truth: 993262,992665,991098,996599,992247,997534,990818,990802,992701,995626,990402,999466,991236,997442,998303,991037,996216,997897,994310,998299,997825,995516,999435,992573,990630,994098,997334,991212,995991,994964\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 5\n", + "Returned IDs: 995274,997320,998779,992994,999144,992630,999026,998184,998365,995755,998951,999090,994100,993412,994964,998862,994757,997976,995336,990949,994730,994894,995438,997737,991790,999089,993529,997832,995824,993484\n", + "Ground Truth: 995274,997320,998779,992994,999144,992630,999026,998184,998365,995755,998951,999090,999647,994100,993412,996959,998633,997925,994964,995701,998862,994757,995702,997976,995336,990949,997377,993648,994730,994894\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 6\n", + "Returned IDs: 992384,995274,995204,993909,991743,992994,998184,999374,997645,990862,998500,995928,992582,990432,992147,995702,994850,998458,996521,992630,992637,998779,997301,996959,997192,994380,991585,997737,994197,995260\n", + "Ground Truth: 992384,995274,993648,995204,993909,991602,991743,993089,992994,998184,991866,993837,999374,997645,993412,990862,998500,995928,999533,992582,994636,990432,992147,995702,994850,998458,996521,992630,992637,991151\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 7\n", + "Returned IDs: 990956,992052,991185,993948,991451,995184,994488,991981,994742,997897,996277,996911,991111,995047,993991,994185,992973,995408,991769,999767,992595,994743,990336,994280,996328,995873,990887,998045,996657,995251\n", + "Ground Truth: 990956,992052,991185,993948,991451,995184,994488,991981,994742,997897,996220,996277,996911,991111,995047,993991,994185,992973,995408,991769,997059,999767,992595,990364,994743,990336,994280,996328,994134,996043\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 8\n", + "Returned IDs: 997913,998090,990949,990910,997320,995961,999710,997301,990231,996918,990560,996805,993504,998950,999639,998337,995336,997371,999806,994850,992748,994100,993529,998365,993696,991498,990957,992944,995802,999331\n", + "Ground Truth: 997913,998090,990949,990910,997320,995961,999710,997301,990231,996918,993412,990560,996805,993504,993515,998950,992277,999639,998337,995336,997371,999806,994850,992748,994100,993529,999900,998365,999338,991498\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 9\n", + "Returned IDs: 995274,998495,995755,992994,990925,992546,994100,994964,994266,990096,997320,999144,993168,991721,998779,996805,996368,996001,998862,993238,997976,993289,990957,992558,998458,990856,995701,994409,999026,993853\n", + "Ground Truth: 995274,998495,995755,992994,990925,993412,992546,994100,993648,994964,994266,990096,997320,999144,993168,991721,998779,996805,996368,996001,999024,998862,993238,997976,993289,990957,992558,998458,990856,994382\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 10\n", + "Returned IDs: 991067,996273,999530,993530,999193,998316,997057,994677,993223,994301,991190,997723,992545,998650,992493,999961,995963,997675,995723,997824,995685,993455,996046,990930,999646,991052,994948,996139,995507,995101\n", + "Ground Truth: 991067,996273,999530,993530,999193,998316,997057,994677,993223,994301,991190,997723,992545,998650,992493,999961,995963,997675,995723,997098,997824,992582,995685,993455,996046,990930,999646,991052,997083,991840\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 11\n", + "Returned IDs: 995459,992824,997775,990084,996742,991432,996380,991817,995503,990192,998754,993077,993891,992615,998061,990720,999639,997547,993583,990538,999564,994270,991272,990273,991871,997055,992095,998308,991070,998142\n", + "Ground Truth: 995459,992824,997775,990084,996742,991432,996380,991817,995503,990192,998754,993077,993891,992615,998061,990720,999639,997547,993583,990538,999564,994270,991272,990273,991064,991871,997055,992095,998308,991070\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 12\n", + "Returned IDs: 993497,993923,992784,992992,999726,992981,992934,992997,998293,990322,992052,996328,994802,998993,990391,992193,994280,997897,994196,999893,990306,991115,991285,995790,995890,990205,995058,996211,997244,991924\n", + "Ground Truth: 993497,990103,993923,992784,992992,999726,992981,992934,992997,998293,990322,992052,996328,994802,998993,990391,992193,994280,997897,994196,999893,997475,990306,991115,991285,995790,995890,990205,995058,996211\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 13\n", + "Returned IDs: \n", + "Ground Truth: 997135,998837,993237,993708,996100,994534,998057,997255,990043,990064,997275,996402,995210,999898,996419,993570,991801,992907,999592,990580,991840,995788,992328,994318,995657,993748,994722,991407,991999,998782\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 14\n", + "Returned IDs: 992561,995211,997733,990126,999985,990931,998251,999179,995633,994142,996411,990705,993219,992566,994783,995883,999191,990621,996431,994317,992415,998691,994587,996609,990854,990069,997823,997403,990340,995571\n", + "Ground Truth: 992561,995211,997733,990126,999985,990931,998251,999179,995633,994142,996411,990705,993219,992566,994783,998870,995883,999191,990621,996431,994317,992415,998691,994587,993775,996609,990854,990069,999352,997823\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 15\n", + "Returned IDs: \n", + "Ground Truth: 996960,995695,991755,995853,994280,998831,993805,997666,991459,992021,997506,998623,992429,998934,994531,990999,994105,990844,996328,993014,997790,994680,990479,990573,990998,996001,993848,995963,990771,995652\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 16\n", + "Returned IDs: 994222,996659,999277,995289,997406,995401,995144,998999,998045,991449,991463,995200,999770,996867,993433,995359,998530,990940,997723,992160,998297,992947,996302,991942,996769,993649,994409,992125,997319,998776\n", + "Ground Truth: 994222,996659,999277,995289,997406,996864,996015,995401,995144,998999,998045,991449,991463,995200,999770,996867,993433,995359,998530,990940,997723,999587,997473,992160,998297,992947,996302,991942,996769,993649\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 17\n", + "Returned IDs: 995886,991743,995893,998921,996351,994982,995515,991692,997218,996762,995968,990215,990561,999698,993250,994334,995154,997301,997002,993143,999075,990995,997571,993820,993237,998908,992384,997320,996903,992790\n", + "Ground Truth: 995886,991743,995893,998921,994850,996351,994982,995515,991692,997218,996762,994810,995968,990215,990561,999698,993250,994334,995154,999823,998445,997301,996438,997002,993143,999075,990995,997571,997370,993820\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 18\n", + "Returned IDs: 991640,996209,999909,999555,996566,997666,999576,990323,998006,997242,997685,995385,996733,995344,993513,991667,990173,993508,991385,995214,994727,994467,999246,991596,991093,994335,990627,996809,994181,990194\n", + "Ground Truth: 991640,996209,999909,999555,996566,997666,996702,999576,990323,998006,997242,997685,995385,996733,995344,993513,996454,991667,990173,990712,993508,991385,995214,994727,994467,999246,991596,991093,994335,990627\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 19\n", + "Returned IDs: 997370,993018,991437,997320,996001,996309,999586,998050,996810,993642,990532,990843,993840,994081,991943,991002,991787,998495,996219,990772,994964,994168,990974,993484,999977,998779,992792,999639,993053,993696\n", + "Ground Truth: 990338,997370,995899,993018,991437,997320,996001,996309,999586,998050,991224,996810,993642,990532,990843,993840,994081,991943,991002,991787,998495,996219,990772,994964,992655,991019,994168,995283,999561,992583\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 20\n", + "Returned IDs: 994351,996495,992237,995754,996809,991383,999435,992726,990194,994098,996428,995214,996774,997725,996328,990173,998335,996839,992193,997967,997331,999430,993224,994481,990705,990594,999832,999752,991723,990712\n", + "Ground Truth: 994351,990402,996495,992237,995754,996809,991383,991025,999435,992726,990194,994098,996428,995214,996774,997725,996328,990173,998335,996839,992193,997967,997331,999430,993224,994481,990705,990408,990594,999832\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 21\n", + "Returned IDs: 993018,997320,994171,993662,997913,990843,996075,994604,995596,995266,991142,997470,996296,990957,999077,998540,997457,993696,991999,997707,996469,992733,999784,999806,998950,990778,994440,995365,999426,999109\n", + "Ground Truth: 993018,997320,994171,993662,995351,997913,990843,996075,994604,995596,995266,994746,994366,991142,997470,996296,994341,990957,999077,998540,997457,993696,991999,997707,996469,995560,992733,999784,992277,997022\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 22\n", + "Returned IDs: 997290,998557,997517,994110,998543,991334,994893,995030,995159,996809,999924,997947,999788,995948,992566,996747,995190,990814,994422,998913,992289,990722,999942,992588,994541,998340,992364,992077,996302,995200\n", + "Ground Truth: 997290,998557,997517,994110,998543,991334,994893,995030,995159,996809,999924,997947,999788,994062,995948,992566,996747,995190,990814,994422,998913,992289,992468,990722,994824,999942,998194,992588,994541,998340\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 23\n", + "Returned IDs: 996053,990096,997913,997586,994850,995536,998090,995755,995927,998861,994894,999356,999711,997305,994618,998921,992848,997984,998550,994964,998495,992330,997320,992862,990866,999710,993431,990066,992384,996001\n", + "Ground Truth: 996053,990096,997913,997586,994850,995961,995536,998090,996143,995755,995927,998861,994894,999356,999711,993853,997305,994618,998921,992848,997984,992716,998550,994707,991582,995503,994964,998495,992330,997320\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 24\n", + "Returned IDs: 996093,997852,994922,992169,993717,991473,998523,997395,996438,992632,992152,990580,990399,994510,996727,991840,993143,998458,999676,997476,990080,995295,992813,997192,993781,996406,999453,999076,998671,993355\n", + "Ground Truth: 996093,997852,994922,992169,993717,991473,998523,997395,996438,992632,992152,990580,990399,994510,996727,991840,993143,998458,999676,997476,990080,991609,993859,995295,992813,997192,993781,996406,992044,999453\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 25\n", + "Returned IDs: \n", + "Ground Truth: 993412,998045,994419,996977,996818,996360,994057,995923,992129,992089,995702,999940,996769,994549,999450,998906,992413,991958,995289,991585,992660,991996,999530,993199,996867,995547,998545,996242,990975,999868\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 26\n", + "Returned IDs: 990487,993500,991977,990998,994254,992558,991447,993335,998686,992052,994148,996068,990684,998406,993975,992170,991620,994006,993583,998536,995408,992718,996262,998623,992280,993965,994080,993715,993681,994280\n", + "Ground Truth: 990487,993500,991977,990998,994254,992558,991447,993335,998686,992052,994148,996068,990684,998406,993975,992170,991620,994006,993583,998536,995408,998480,992718,996262,992933,992554,998623,992280,993965,994080\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 27\n", + "Returned IDs: 993721,996013,991409,993053,991382,999352,994983,998568,998495,993984,997270,996075,995596,993379,999806,999639,996650,994886,995535,994431,996001,997442,999302,993342,995352,996328,992515,993621,991128,995476\n", + "Ground Truth: 993721,996013,991409,991179,993053,991382,999352,994355,994983,996219,998568,998495,993984,997270,996075,997547,995596,993379,999806,999639,996650,994886,997448,995535,994431,996001,997442,999302,993342,995352\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 28\n", + "Returned IDs: 990615,998937,994703,996840,995788,994216,996915,991687,992662,999676,995515,990475,994601,998244,993495,991840,993190,992835,994612,998523,991485,990965,995080,993905,991392,997403,999613,998139,997220,994033\n", + "Ground Truth: 990615,998937,994703,996840,995788,994216,996915,991687,992662,999676,995515,990475,994601,998244,993495,991840,993190,992835,994612,998523,991485,990965,991499,995080,993905,993028,997496,991392,997403,999613\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 29\n", + "Returned IDs: 998276,993603,994829,997918,997002,995141,996996,996012,993139,996661,993289,991425,990051,996808,997977,992330,993504,994972,996576,994946,998401,992225,995855,997777,992979,990491,999111,994821,996001,996871\n", + "Ground Truth: 998276,993603,994829,997918,997002,995141,996996,992047,996012,993139,996661,993289,991425,997140,990051,996808,997977,992330,993504,992238,994972,996576,994946,998401,992225,994042,995855,997777,992979,990491\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 30\n", + "Returned IDs: 995257,992888,990402,997673,998570,997666,990158,996702,995231,995695,997369,992664,995349,997086,995393,999435,990041,990479,995142,998006,990194,991640,996960,994467,993461,996594,992945,995168,991755,993067\n", + "Ground Truth: 995257,992888,990402,997673,998570,997666,990158,996702,995231,992209,995695,997369,992664,995835,995349,997086,995393,999435,990041,990479,995142,998006,990194,991640,996960,994467,993461,996594,992945,995168\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 31\n", + "Returned IDs: 990811,998593,995526,992334,996298,997732,998039,992684,997969,990089,996834,990529,993642,994964,994783,995126,995679,997410,996001,999099,994925,992120,996873,993831,994293,999302,998382,990576,995861,993561\n", + "Ground Truth: 990811,998593,995526,992334,996298,997732,998039,992684,997969,997112,990089,996834,990529,993642,994964,994783,995126,995679,997410,995906,996001,999099,994925,992120,996873,993831,994293,999302,998382,991200\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 32\n", + "Returned IDs: 993781,999077,996592,998123,997918,993260,994552,998950,990580,995357,991791,990043,991157,992571,998180,998951,991015,991435,994964,995336,996196,993512,999144,994972,996658,991550,990339,992692,999676,993453\n", + "Ground Truth: 993781,999077,996592,992637,998123,997918,993260,999701,994552,998950,990580,995357,991791,990043,991157,992571,993633,995584,998180,998951,991015,991435,994964,995336,996196,993512,999144,994972,996658,990885\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 33\n", + "Returned IDs: 995211,994783,994587,994665,998913,995289,992827,995633,997823,992488,992219,993209,998823,996609,997209,995348,998188,999627,999957,993586,991269,991507,999759,992415,993857,997203,993642,995558,998396,992566\n", + "Ground Truth: 995211,994783,994587,994665,998913,995289,992827,995633,997823,992488,992219,993209,998823,996609,997209,995348,998188,999627,999957,993586,991269,991507,999759,992415,993857,997203,995401,993642,995558,998396\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 34\n", + "Returned IDs: \n", + "Ground Truth: 995130,994659,995332,993808,995750,997273,996128,997542,991961,997412,993347,993079,993304,991990,998860,997442,994452,994871,991343,998891,997199,993931,992567,995107,993499,992358,991061,993324,997555,993548\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 35\n", + "Returned IDs: 994397,996075,995274,991422,999289,995336,997571,993168,993280,999596,990585,997976,996108,994432,991125,990862,996714,997304,997377,994894,996368,993167,995502,995555,994100,992880,991721,997015,998779,990990\n", + "Ground Truth: 994397,996075,995274,991422,999289,996976,998011,995336,997571,993168,995374,990701,993280,999596,997759,990585,997976,996108,994432,991125,990862,996714,997304,994916,997377,994894,996368,999437,993167,997925\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 36\n", + "Returned IDs: \n", + "Ground Truth: 993381,996395,991513,998586,992721,999994,991945,992136,991610,990991,990177,997145,999555,990167,990292,991987,993122,997149,997009,995257,997506,998006,996623,995655,991038,995780,996571,990330,992444,991835\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 37\n", + "Returned IDs: 998143,991795,993379,995864,996759,993003,998205,997909,992448,996719,997868,993269,996256,994409,991969,997059,996162,998361,993561,991175,994140,998509,993537,998910,996187,996987,991357,994076,999362,991197\n", + "Ground Truth: 998143,991795,993379,995864,996759,993003,998205,997909,992448,996719,997868,993269,996256,994409,991969,997059,996162,998361,993561,991175,994140,998509,993537,998910,996187,996987,991357,994076,999866,999362\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 38\n", + "Returned IDs: 990210,999374,993860,999331,990545,999210,994964,995438,992147,994336,995928,992944,998302,995927,991743,993018,993484,991801,996805,997320,991498,999898,993877,992169,996219,997002,990083,990096,999122,990843\n", + "Ground Truth: 990210,999374,999338,993860,998160,999331,990545,999210,991720,994964,995438,992147,998294,994336,990648,995928,992944,993154,998302,995927,991743,993018,993484,991801,995075,996805,993238,997320,991498,999898\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 39\n", + "Returned IDs: 993433,997406,998165,995289,992125,993390,996112,996042,994419,992160,992576,996277,997822,991415,998564,996867,995439,992519,992925,996769,998045,997766,991145,992556,996322,997046,994571,990836,998486,995200\n", + "Ground Truth: 993433,997406,998165,995289,992125,993390,992881,996112,996042,994419,995401,992160,992576,996277,997822,991415,998564,996867,995439,992519,992925,996769,998045,997766,991145,996294,992556,996322,997046,994571\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 40\n", + "Returned IDs: 993484,997320,998246,996296,997370,995949,996452,997913,993885,990772,990957,997661,998302,990843,991801,993018,991498,990375,998950,993803,995932,993741,994366,997470,990949,991721,996143,994089,993154,995596\n", + "Ground Truth: 993484,997320,998246,996296,997370,995949,996452,997913,993885,990772,990957,997661,998302,990843,991801,993018,991498,990375,998950,998160,993803,995932,993741,994366,997470,990949,991721,996143,995265,991063\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 41\n", + "Returned IDs: \n", + "Ground Truth: 996395,990194,999555,998006,997205,997242,999994,999576,995167,990167,994467,993106,998122,990360,990230,993381,997149,999435,996209,991148,992771,993122,997086,998066,991767,993224,998429,994796,994638,995848\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 42\n", + "Returned IDs: 998951,993536,992646,993492,999144,992330,992969,992909,995833,994670,995336,992452,999089,993544,990275,996646,997918,998465,992183,998011,997192,999374,992384,997875,994100,996592,995793,994403,998401,994767\n", + "Ground Truth: 998951,993536,992646,993492,999144,992330,992969,992909,995833,994670,995336,992452,999089,993544,990275,996646,997918,998465,995374,992183,990171,998011,997192,999374,992384,997875,994100,996592,999343,992932\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 43\n", + "Returned IDs: 994541,999372,999830,995300,998055,998671,994322,994557,995280,992434,993437,995861,993929,993939,998484,995704,996057,990931,994558,991327,994119,993856,995191,993877,994782,998710,999416,996816,994692,994035\n", + "Ground Truth: 994541,999372,999830,995300,998055,998671,994322,998466,994557,995280,996939,992434,993437,995750,995861,993929,993939,998484,995704,991893,996057,990931,991537,994558,991327,994119,993856,995191,993877,994782\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 44\n", + "Returned IDs: 997320,990375,992041,993018,990843,994171,998950,991498,998246,990949,993785,998540,993484,997305,994081,991650,996075,992880,993260,993154,990083,997470,995438,999701,997370,996296,993504,999453,991860,991999\n", + "Ground Truth: 997320,990375,992041,993018,990843,994171,998950,991498,998246,990949,993785,999338,998540,993484,997305,994081,991650,996075,992880,993260,993154,990083,997470,995438,996143,999701,997370,996296,993504,999453\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 45\n", + "Returned IDs: 996001,990366,995536,998720,999216,992862,998934,993139,996500,993289,993055,993174,990449,994829,994672,997586,990399,993560,995672,995398,994167,999806,999639,997002,998593,995418,990352,990891,999111,993696\n", + "Ground Truth: 996001,990366,995536,998720,999216,992862,998934,993139,996500,993289,993055,993174,990449,994829,994672,997586,990399,993560,997531,995672,995398,994167,999806,999639,997002,998593,995418,990352,992617,990891\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 46\n", + "Returned IDs: 996504,998921,997889,998170,997192,994775,992147,990647,991743,995515,993518,996586,991127,999374,994939,996218,990515,991782,993250,992087,998721,999327,990275,998401,994919,999263,995621,995985,991479,993443\n", + "Ground Truth: 996504,998921,997889,990917,998170,997192,994775,992147,990647,991743,995515,993518,996586,991127,999374,994939,996218,990515,992379,991782,993250,992087,998721,999327,990275,992789,998401,994919,999263,995415\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 47\n", + "Returned IDs: 991403,999595,990325,993425,994446,999277,995144,997766,990751,997406,998999,990116,994222,995940,996302,995701,991478,998530,995401,997723,991630,991942,991603,996388,995478,995182,998045,995665,990242,999977\n", + "Ground Truth: 991403,999228,999595,990325,993425,994446,999277,995144,996015,993137,997766,990751,997406,998999,996864,990116,994222,995940,996302,995701,991478,998530,995401,997723,991630,991942,991603,995994,996388,995478\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 48\n", + "Returned IDs: 994925,993561,995895,991958,998382,998495,992448,993906,998593,993053,992521,999294,996219,998106,993637,997386,996001,992847,998649,994742,994419,995200,996397,998039,990435,997822,997529,996042,990096,992944\n", + "Ground Truth: 994925,995873,993561,995895,991958,998382,998495,992448,993906,998593,993053,992521,992120,999294,996219,998106,993637,994657,997386,996001,992847,998649,994742,997379,994419,995200,995899,996397,995401,998039\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 49\n", + "Returned IDs: 996523,992558,995552,990989,995599,995972,992411,997054,993534,999715,991550,996606,999860,996196,999564,993427,997523,990279,997681,998220,996655,994384,997547,990181,998417,993792,999540,993474,991953,997739\n", + "Ground Truth: 996523,992558,995552,990989,995599,995972,992411,997054,993534,999715,991550,996606,999860,996196,999564,993427,997523,990279,994006,997681,994404,998220,996655,994384,997547,990181,991595,998417,992141,993792\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 50\n", + "Returned IDs: 991081,992664,998076,993715,998716,994131,997230,993917,997197,997097,998949,996858,991663,992495,991684,996021,998440,990622,992735,998030,996183,998446,999139,991524,996986,995735,997673,993366,998812,993335\n", + "Ground Truth: 991081,992664,998076,993715,998716,994131,997230,993917,997197,997097,998949,996858,991663,992495,991684,996021,998440,990622,992735,998030,996183,998446,994918,999139,991524,998845,998680,991021,996986,995735\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 51\n", + "Returned IDs: 998794,993984,993184,994654,996739,998498,991877,999323,995596,994458,994886,993721,997661,994985,996219,995352,991409,997270,999722,993815,998588,994440,996936,994873,996616,999853,999430,999473,999551,997555\n", + "Ground Truth: 998794,994452,993984,993184,994654,996739,998498,991877,990780,999323,995596,994458,994886,993721,997661,994985,996219,995352,991409,997270,999722,993815,998588,990482,994440,996936,995622,998580,994873,996616\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 52\n", + "Returned IDs: 990096,998934,996001,998720,997906,998697,993174,998861,994167,997685,996164,993275,993154,996124,995536,999736,995203,992429,999639,998961,994718,993744,999831,999367,990991,996230,999499,998593,996893,995780\n", + "Ground Truth: 990096,998934,996001,998720,997906,998697,993174,998861,995855,994167,997685,996164,993275,993154,999343,996124,995536,999736,996894,999647,995203,992429,999639,999882,995401,998961,998514,994718,993744,999831\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 53\n", + "Returned IDs: 996436,992912,997640,990797,998779,991750,997355,993412,999144,993526,993725,995755,999740,997256,994990,995923,994436,994894,990388,999026,994964,996714,994838,999021,991173,997192,999916,990845,999004,995701\n", + "Ground Truth: 996436,992912,997640,990797,998779,991750,997355,993412,999144,991651,999587,993526,993725,995755,999740,997256,994990,994916,995923,994436,994894,990535,990388,999026,994964,996714,994838,999021,991173,999344\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 54\n", + "Returned IDs: \n", + "Ground Truth: 997320,999639,993738,990930,990532,995873,997822,996371,995395,996714,992546,990878,993293,992977,992521,991721,995375,999026,999527,990634,996777,990814,994140,991349,990585,992318,991224,995274,995813,993561\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 55\n", + "Returned IDs: 996627,992079,992680,994751,993205,996411,990528,993725,999302,999708,999740,991382,997933,992230,994783,995633,998055,990985,991669,992652,995456,995786,993053,991464,999126,995756,995076,994472,994567,993240\n", + "Ground Truth: 996627,992079,992680,994751,993205,996411,990528,997207,993725,999302,999708,999740,991382,997933,990743,992230,994783,995633,998055,990985,991669,991537,992652,995456,995786,993053,991464,999126,995756,995076\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 56\n", + "Returned IDs: \n", + "Ground Truth: 998416,991996,997014,997920,998221,997804,997900,995786,990126,994964,993378,997403,998495,991377,996305,990931,997477,991565,994466,992446,993199,998816,990158,996219,991179,997822,994419,993561,997586,993444\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 57\n", + "Returned IDs: 994724,998257,995207,999834,995936,998078,991987,992406,990450,996488,991513,995257,992856,999808,998959,996393,990657,998918,999994,992762,998005,992760,997801,995056,993115,999555,999117,993332,992623,990777\n", + "Ground Truth: 994724,998257,995207,999834,995936,998078,991987,992406,990450,996488,991513,995257,993381,992856,999808,998959,993884,996393,990657,998918,999994,992762,998005,992760,997801,996395,995056,993115,999555,999117\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 58\n", + "Returned IDs: 996196,990159,996690,997224,996523,994384,993670,994372,998286,990303,990824,996655,995503,991550,995290,990989,999860,994950,998411,994964,994912,993158,990538,995599,991458,995410,998697,991791,999716,998230\n", + "Ground Truth: 996196,990159,996690,997224,996523,994384,993670,994372,998286,990303,990824,996655,995503,991550,995290,990989,999860,994950,996744,998411,994964,994912,993158,990538,995599,991458,995410,998697,991791,999716\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 59\n", + "Returned IDs: 991451,991650,999698,995357,993721,993215,997002,994226,998951,993154,991997,992557,998961,992880,994449,990532,997918,996685,990083,994081,996178,994743,993018,995401,991535,999453,999831,999442,997571,995372\n", + "Ground Truth: 996280,991451,995374,991650,999698,995357,993721,994030,993215,997002,994226,998951,993154,991545,995181,991997,992557,999900,994850,998961,992880,994449,990532,997448,997918,994362,996685,990083,994081,993512\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 60\n", + "Returned IDs: 998224,999496,995831,995196,995407,994956,999138,994817,999244,990678,998779,995295,998691,996328,995861,993725,991319,991378,997732,992713,993678,996918,997714,995020,998067,994587,991192,990392,994119,994765\n", + "Ground Truth: 998224,999496,995831,995196,995407,994956,995087,999138,994817,999244,990678,998779,995295,998021,998691,996328,995861,993725,991390,991319,991378,997732,992713,992703,993678,996918,997714,995020,998067,994587\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 61\n", + "Returned IDs: 993738,992651,998831,995701,995184,991836,998697,997059,998060,993280,995079,995873,996828,992609,991530,996657,992293,995640,993948,991175,991007,997897,995216,994488,996001,990005,997554,992052,991576,998549\n", + "Ground Truth: 993738,992651,998831,995701,995184,991836,998697,996534,997059,998060,993280,995079,995873,996828,992609,991530,996657,992293,995640,993948,991175,991007,997897,995216,994488,996001,994163,990005,997554,992052\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 62\n", + "Returned IDs: 991157,999122,991015,998607,997305,992718,997192,993143,998180,996193,998123,995515,994046,992052,993260,998170,996504,992147,998950,991848,993077,998921,995408,995372,998106,993609,995154,999676,993781,998827\n", + "Ground Truth: 991157,999122,991015,997305,998607,991720,992718,997192,993143,998180,996193,998123,995515,994046,992052,997496,993260,998045,998170,996504,992147,998950,991848,995401,993077,998921,995408,995864,995372,998106\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 63\n", + "Returned IDs: 995059,990264,990516,991590,991960,998999,992415,995020,998525,993293,993079,990521,994435,995359,993180,992171,999981,995421,997319,994697,997718,990610,994349,992239,994409,991245,991867,991775,998961,994040\n", + "Ground Truth: 995059,990264,990516,991590,991960,998999,992415,995020,998525,993293,993079,990521,992318,994435,995359,993180,992171,999981,995421,997319,994817,994697,997718,990610,994349,992239,994409,991245,991867,991775\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 64\n", + "Returned IDs: 996184,999634,992754,996357,991089,991005,999722,993335,994958,995191,996794,995131,998794,995880,994370,992628,991382,995313,993479,998716,994769,998192,993073,998949,996153,999430,997331,996016,991554,999920\n", + "Ground Truth: 996184,999634,992754,996357,991089,991005,999722,993335,994958,995191,996794,995131,998794,995880,994370,992628,991382,995313,993479,998716,990402,994769,998192,993073,998949,996153,999430,997331,996016,991554\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 65\n", + "Returned IDs: 996809,998695,991571,993617,998790,999009,998786,991890,995461,990288,996370,999711,999430,998849,990538,995375,999194,990489,994451,997821,993515,995401,990930,996830,995560,991532,994643,993696,995559,996284\n", + "Ground Truth: 996809,998695,991571,993617,998790,999009,998786,991890,995461,990288,996370,999711,999430,991139,998849,990361,990538,995375,999194,990489,991052,994451,997821,993515,995401,992452,990930,996830,995560,991532\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 66\n", + "Returned IDs: 999526,997970,991245,999281,991629,997550,994016,997584,996075,993715,992351,992825,995375,995740,992747,993504,995372,998097,999724,993609,994531,996001,992052,992636,990096,992521,999338,999840,995203,994233\n", + "Ground Truth: 999526,997970,991245,999281,991629,997550,994016,997584,996075,993715,992351,992825,995375,995740,992747,993504,995372,998097,997691,999724,993609,992933,994531,996001,995727,994610,992052,992636,990096,992521\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 67\n", + "Returned IDs: 992664,994131,998446,998030,998716,991684,991486,993931,997453,996147,991663,998416,991081,996328,998949,996733,998440,993335,998536,998903,996830,998076,997073,993917,997944,991961,992408,993715,996858,990712\n", + "Ground Truth: 992664,994131,998446,998030,998716,997097,991684,991486,993931,997453,996147,991663,998416,991081,996328,998845,998949,996733,998440,993335,998536,998903,992780,996830,998076,997073,993917,997944,996028,991961\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 68\n", + "Returned IDs: 997547,992736,999908,998933,990538,995737,992824,994165,993706,997215,992170,990273,991070,995372,990192,997914,997775,999639,991432,991272,990279,995459,996196,993055,993611,993077,990885,996380,990930,997614\n", + "Ground Truth: 997547,992736,999908,998933,990538,995737,992824,994165,993706,997215,992170,990273,991070,995372,990192,997914,997775,999639,991432,997506,991272,990279,995459,996196,993055,993611,993077,997301,997821,990885\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 69\n", + "Returned IDs: 994100,993492,991673,996805,992330,995336,998495,995833,995968,990231,998921,992944,993512,995357,995748,996001,995274,998090,993154,995536,996219,997605,995200,997918,996790,996592,990957,999544,992443,992318\n", + "Ground Truth: 994100,993492,991673,996805,992330,995336,998495,994334,995833,995968,990231,998921,992944,990271,996769,993512,995357,995748,996001,995274,998090,993154,994088,995536,995826,996282,995162,995401,990648,996219\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 70\n", + "Returned IDs: 990086,998859,995851,996991,998748,997031,993908,998181,999805,990903,995268,997185,995778,994991,995106,993320,993466,998980,994945,991815,990435,999402,992805,990751,993883,998073,994759,995086,995324,998067\n", + "Ground Truth: 990086,992540,998859,995851,996991,998748,997031,993908,998181,999805,990903,995268,997185,995778,994991,995106,993320,993466,998980,994945,991815,990435,999402,992805,990751,993883,998073,994759,995086,995324\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 71\n", + "Returned IDs: 995841,995375,995372,992944,999711,996318,990043,994972,990585,997301,996193,999544,991977,997320,995560,996001,998180,995268,998097,995138,996609,992384,995515,991743,996219,992047,998401,993324,998921,990334\n", + "Ground Truth: 995841,995375,995372,992944,999711,996318,990538,990043,994972,998726,991052,995401,990585,997301,998445,996193,999544,991977,991540,997320,995560,996001,998180,995268,998097,995138,996609,992384,993533,999526\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 72\n", + "Returned IDs: 993738,995395,992790,990930,993409,997928,997320,991576,991349,997027,990784,996714,995007,992036,997218,997675,997168,996247,995701,998132,996918,994488,995521,992179,990561,998303,994170,994906,998723,995268\n", + "Ground Truth: 993738,995395,992790,990930,993409,997928,997320,991576,991349,997027,990006,990784,996714,995007,999820,992036,997218,995375,997675,997168,996247,995701,998132,995807,996918,994488,995521,992179,990561,998303\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 73\n", + "Returned IDs: 993113,994817,992832,991629,999827,993324,992636,994119,993917,995332,993533,997357,990993,990889,990392,996986,990814,992567,997309,993715,992026,990289,995746,993456,993304,999796,999791,992764,993742,995012\n", + "Ground Truth: 993113,996810,994817,992832,991629,999827,990468,999942,993324,992636,994119,993917,995332,993533,997357,990993,990889,990392,996986,990814,992567,997309,993715,992026,990289,995970,998848,995746,993456,993304\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 74\n", + "Returned IDs: \n", + "Ground Truth: 994102,994831,998764,995372,997913,991362,999982,991335,995374,998702,992880,992291,990585,993852,996143,995828,990538,990724,992748,994618,990949,997470,992114,990957,990275,996959,998500,998744,997192,996785\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 75\n", + "Returned IDs: 998981,998287,997320,998246,994225,996296,994889,997178,994450,991155,990020,992880,994214,994249,997470,998540,993484,992748,994542,993529,996075,996085,994416,990742,994964,995400,998550,993662,992811,990532\n", + "Ground Truth: 998981,998287,997320,998246,994225,996296,992902,994889,997178,994450,991155,990020,994214,992880,994249,997470,995116,998540,993484,992748,994542,998634,993529,996075,996085,995700,994416,999132,990742,994964\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 76\n", + "Returned IDs: 991212,995167,990557,993855,996001,994351,999880,990712,995214,996328,990752,993931,999435,993715,994280,990705,990314,999502,997897,997790,992864,993181,995873,991257,997002,991605,992726,991098,991184,999139\n", + "Ground Truth: 991212,995167,990557,993855,993981,996001,994351,999880,990712,995214,996328,990752,993931,997860,999435,993715,994280,990705,990314,999502,997897,997790,992864,993181,995873,991257,997002,991605,992726,991098\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 77\n", + "Returned IDs: 992713,998671,999830,994541,998279,990931,998256,994142,990705,991319,996189,990364,996328,992445,991327,991058,994891,998543,999191,994035,996816,991682,995191,996647,990712,997967,994098,999179,999775,999458\n", + "Ground Truth: 997669,998466,992713,998671,999830,994541,998279,990931,998256,994142,990705,991319,996189,990364,996328,992445,991327,991058,994891,998543,999191,994035,996816,997933,991682,995191,992780,996647,990712,997967\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 78\n", + "Returned IDs: 995618,990124,996158,996367,992306,993534,994398,994028,997505,996953,994080,999908,999022,996585,997467,998357,996606,992092,994760,991977,993427,995965,993583,990308,994679,992558,994011,993681,998652,992857\n", + "Ground Truth: 995618,990124,996158,996367,992306,993534,994398,994028,997505,996953,994080,999908,999022,996585,997467,993323,998357,996606,992092,994760,991977,993427,995965,993583,990308,994679,992558,994011,993491,993681\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 79\n", + "Returned IDs: 992931,997843,992514,996993,991260,998764,997483,995570,993484,991131,997913,990949,997320,996378,998195,999982,992748,998653,992105,992781,991872,992069,991752,995336,995596,995261,993492,994757,992980,993154\n", + "Ground Truth: 992931,997843,992514,990742,996993,991260,998764,997483,995570,990478,992961,993484,991131,997913,990949,997320,996378,998195,999982,992748,999838,999855,998653,992105,992781,991872,992069,995277,991752,995336\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 80\n", + "Returned IDs: 995841,995372,996785,997301,994964,996959,997796,991444,998921,997320,999374,990560,990585,992147,996218,998839,995949,995886,994972,991743,998797,992066,996805,990925,992944,998302,997015,996052,999263,997192\n", + "Ground Truth: 995841,995372,996785,998445,997301,994964,996959,997796,991444,998921,997320,999374,998644,990560,999533,990585,992147,993238,992789,996218,998839,993648,995949,996090,995886,994972,991743,998797,992066,996805\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 81\n", + "Returned IDs: 990146,993237,992085,997747,994318,992944,991727,991392,995443,990142,993253,997591,997585,990742,991121,994789,999784,998782,990822,994534,993570,990615,998550,990447,999676,999948,990650,992218,998139,990648\n", + "Ground Truth: 990146,993237,992085,997747,994318,992944,991727,992342,991392,995443,990142,993253,997591,997855,997585,990742,991121,994789,999784,998782,990822,995334,994534,993570,990615,998550,990447,999676,999948,996352\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 82\n", + "Returned IDs: 998874,996505,992521,996001,995110,997675,998536,995521,996715,990537,995873,990930,993931,996028,997067,996918,998593,993289,998397,997822,995268,992703,995536,996368,992488,992448,994380,996930,995701,992790\n", + "Ground Truth: 998874,996505,992521,995401,996001,995110,997675,998536,995521,996715,990537,995873,990930,995130,993931,996028,997067,996918,991459,998593,993289,998397,997822,995268,992703,995536,991052,996368,992488,992448\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 83\n", + "Returned IDs: 998779,994436,997640,997536,991028,991350,990406,991603,995702,997058,994161,997787,996379,997355,991958,998165,996388,994396,995052,990845,991011,993813,996719,990984,996042,993040,993831,997822,999196,996125\n", + "Ground Truth: 998779,994436,997640,997536,991028,991350,990406,991603,995702,997058,994161,997787,996379,997355,991958,998165,996388,994396,995052,990845,991011,993813,996719,990984,998999,996042,993040,993831,997822,999196\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 84\n", + "Returned IDs: 997913,994964,997371,993237,995515,994778,996727,996811,990142,996840,991727,995928,998359,996001,996438,994277,994415,997593,997291,999426,993143,996592,999290,995886,990580,997320,997370,994360,997301,990066\n", + "Ground Truth: 997913,994964,997371,993237,995515,994778,994475,991801,996727,997350,996811,990142,996840,991727,995928,998359,996001,996438,994277,994415,997593,997291,999426,992140,993143,994971,996592,999290,995886,990580\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 85\n", + "Returned IDs: 995207,991701,996395,998586,991987,996623,992136,995056,999117,995373,994063,990292,994413,998858,998257,994451,998066,991052,999249,994092,999096,997110,998659,992604,993115,997918,999834,994753,992638,999101\n", + "Ground Truth: 995207,991701,996395,998586,991987,996623,992136,995056,999117,995373,994063,990292,994413,998858,998257,994451,998066,991052,994187,999249,994092,995617,999096,997110,998659,992604,993115,997918,999834,994753\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 86\n", + "Returned IDs: 992752,998933,995011,995945,993037,998536,993931,990407,993055,995421,998341,992437,998250,998594,999139,996483,997926,998593,991245,998303,997019,991530,999111,994463,997506,996775,991633,991552,994259,996028\n", + "Ground Truth: 992752,998933,995011,995945,993037,998536,993931,991291,990407,993055,995421,998341,992437,998250,998594,999139,996483,997926,998593,991245,998303,997019,999265,991530,999111,994463,997506,996775,997531,991633\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 87\n", + "Returned IDs: 990146,991498,999263,998302,992994,990560,999533,994964,998644,998921,997301,998445,991444,990958,995306,995771,990925,993885,990096,999374,996785,992147,992546,994281,994922,994978,991406,999197,990224,996504\n", + "Ground Truth: 990146,997876,991498,999263,998302,992994,990560,999533,994964,993484,998644,998921,997301,998445,991444,990958,995306,995771,990925,993885,992814,990096,999374,996785,992147,992546,994281,994922,994978,991406\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 88\n", + "Returned IDs: 992931,995319,995942,997147,991593,997913,990922,992964,991250,993031,990114,993606,999559,995632,998899,996508,999945,994668,992255,993278,995984,990949,992228,998619,991502,990742,994633,997946,991456,995774\n", + "Ground Truth: 992931,995266,995319,994052,995942,997147,991616,991593,992954,997913,990922,992964,991250,993031,990114,993606,990653,996629,999559,995632,998899,996508,999945,994668,992255,993278,998562,995984,990949,992228\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 89\n", + "Returned IDs: 995755,997437,993484,995244,998649,994894,992296,997320,999740,998246,991991,994814,995435,994359,999599,997304,998855,993696,996053,997305,995841,990949,990786,996075,999867,992924,990307,993529,993818,990957\n", + "Ground Truth: 995755,997437,993630,993484,995244,998649,994894,996630,992296,997320,999740,991999,998246,991991,994814,995435,994359,999599,997304,998855,999708,993696,996053,996436,997305,995841,990949,990786,993412,996075\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 90\n", + "Returned IDs: 999451,997739,994852,997903,990376,992558,996693,995983,997301,995580,996894,998686,991923,999807,999639,997106,992383,998600,994195,994979,998079,999561,996262,992423,992718,990383,993283,996196,992521,994163\n", + "Ground Truth: 999451,997739,994852,997903,990376,992558,996693,995983,997301,995580,996894,998686,991923,999807,999639,997106,992383,998600,992612,994195,994979,998079,999561,996262,992423,992718,990383,993283,996196,992521\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 91\n", + "Returned IDs: 998105,994307,993053,997790,991021,993906,996028,995823,996147,990096,995586,995295,993289,992480,995456,998677,999444,994689,996062,996001,991685,994280,992408,990532,992072,993433,998382,993561,994531,993809\n", + "Ground Truth: 998105,994307,993053,997790,991021,993906,996028,995823,991451,996147,990096,995586,995295,993289,992480,995456,995401,998677,999444,994689,996062,996001,991685,994280,993778,992408,990532,992072,993433,998382\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 92\n", + "Returned IDs: 991148,992551,993472,990194,996216,995895,998122,998495,992892,997506,992209,990096,991411,997442,993184,995798,994661,999576,991382,997685,994682,996184,991881,993460,995535,992780,999430,990091,996001,999139\n", + "Ground Truth: 991148,992551,994355,993472,990194,996216,995895,999336,998122,998495,992892,997506,992209,990096,991411,997442,993184,995798,994661,999576,993121,991382,997685,994682,996184,991881,993460,997196,995535,992780\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 93\n", + "Returned IDs: 998779,991743,996727,993412,996976,992087,995274,997640,999026,998649,999074,992511,997355,995034,992685,998862,992147,998495,990406,992630,995755,996918,990949,999179,990606,994894,990957,999021,996890,990925\n", + "Ground Truth: 998779,991743,996727,993412,996976,990401,993673,991268,992087,995274,997640,999026,998649,999074,992511,997355,995034,992685,998862,997429,992147,998495,990406,994301,995616,996438,997799,992637,992630,995755\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 94\n", + "Returned IDs: 993785,997661,998921,992041,995515,995250,999806,995596,998498,992119,993447,990697,997550,997305,996075,992635,991984,991157,995372,994081,996001,993257,997320,993904,995203,996755,999426,993820,992944,996805\n", + "Ground Truth: 993785,997661,998921,992041,995515,995250,999806,995596,998498,992119,993447,990697,997550,997305,996075,992635,993022,991984,991157,992384,995372,994081,996001,993257,995622,997320,993904,995203,996755,999426\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 95\n", + "Returned IDs: 994964,993143,997320,996001,996657,996500,999219,992384,996219,999343,995928,991459,999144,991392,990999,999653,998303,996028,990366,998246,993174,993168,990771,998593,996576,998302,994280,996368,996147,993484\n", + "Ground Truth: 994964,993143,995855,997320,996001,996657,996500,999219,997920,992384,996219,999343,998029,995928,991459,999144,991392,990999,999653,998303,996028,990366,998246,993174,993168,990771,998593,996576,998302,994280\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 96\n", + "Returned IDs: 993741,992041,996909,991650,993785,995596,996741,990697,997836,992671,992635,999426,997661,994971,991372,995515,993904,996075,996875,997305,995372,990142,992733,998227,999439,993022,998498,990914,997291,994415\n", + "Ground Truth: 993741,992041,996909,991650,993785,995596,996741,990697,991218,997836,992671,992635,999426,997661,994971,991372,995515,993904,996075,996875,997305,995372,990142,992733,998227,999439,995995,993022,998498,990914\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 97\n", + "Returned IDs: 995357,991743,996658,994197,997571,998011,995336,994819,995274,993145,991131,991094,996976,996727,998779,990647,998495,999470,991361,992646,996714,999144,994334,994100,992718,990231,999089,991908,999021,999331\n", + "Ground Truth: 995357,995374,991743,996658,994197,997571,998011,995336,994819,995274,993145,991131,991094,996976,996727,998779,990647,998495,999470,991361,993096,992646,996714,995873,999144,994334,994100,992718,990231,999089\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 98\n", + "Returned IDs: \n", + "Ground Truth: 990712,996495,994351,999388,990705,993224,994481,993801,995214,991633,992726,994956,996728,996328,995754,996566,992445,994310,994378,992780,992154,990194,991296,994047,997086,993931,999852,992429,998389,996216\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 99\n", + "Returned IDs: 998697,999129,998536,991530,995536,993609,995672,992862,999639,995780,992268,991332,999806,997506,992617,990399,993055,995562,997775,990096,998303,999744,996001,995237,992423,996769,991157,994167,993342,998045\n", + "Ground Truth: 998697,999129,998536,991530,995536,993609,995672,992862,999639,995780,992268,991332,999806,997506,992617,990399,991630,993055,995562,999587,997775,990096,998303,996368,999744,995401,996001,995864,995237,992167\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 100\n", + "Returned IDs: 996351,998837,994703,998363,991193,999738,993708,994789,995515,993896,995489,991840,990138,991609,993143,999453,991485,999613,998380,992611,993577,994922,990310,993237,991999,995988,995267,995072,999676,995443\n", + "Ground Truth: 996351,998837,994703,998363,991193,999738,993708,994789,995515,997496,993896,992384,995489,991840,990138,990490,991609,993143,990134,999453,991485,999613,990809,998657,998380,994001,992611,993577,994922,991930\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 101\n", + "Returned IDs: 993813,991790,991163,996077,991466,997046,999731,990940,993561,997630,997289,997386,990751,998633,995679,998165,997822,994409,992919,990930,993293,993738,991958,999037,998593,995401,990634,995701,994140,992703\n", + "Ground Truth: 993813,991790,991163,996077,991466,997046,999731,990940,993561,990878,997630,997289,997386,999000,990751,998633,995679,998165,997822,994409,992919,990930,991603,990820,993293,993738,991958,999037,998593,995401\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 102\n", + "Returned IDs: 999639,998793,992824,996196,997775,994270,992306,990538,992802,999908,996742,994560,990084,997547,992030,995631,994305,995955,997215,990720,997864,990755,993077,999564,992190,991432,992458,991910,992558,994614\n", + "Ground Truth: 999639,998793,992824,996196,997775,994270,992306,990538,992802,999908,996742,994560,990084,997547,992030,995631,994305,991923,995955,997215,990720,999451,997864,990755,993077,999564,992190,990409,991432,990146\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 103\n", + "Returned IDs: 998105,999460,995906,990536,994419,997453,996974,990497,996062,998039,993906,993053,994307,992201,995823,996028,996647,991989,992622,993831,992408,991021,993561,992072,991958,990076,995295,997969,991168,990097\n", + "Ground Truth: 998105,999460,995906,990536,994419,997453,996974,990497,999083,996062,998039,993906,993053,994307,992201,995823,996028,996647,991989,992622,993831,992408,991021,993412,993561,992072,991958,990076,995295,997969\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 104\n", + "Returned IDs: 997554,999453,995216,997937,998495,996147,990845,996001,999402,992979,996368,991585,996810,991790,992792,990441,993561,991967,996759,990662,992364,997031,993289,996911,995289,992521,996657,998060,995963,996987\n", + "Ground Truth: 997554,999453,995216,997937,998495,996147,990845,996001,999402,992979,993412,994531,998166,996368,991585,996810,991790,993003,992792,992334,990441,993561,991943,991967,996759,990662,992364,997031,993289,996911\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 105\n", + "Returned IDs: 997913,991090,993834,999982,999247,992228,998500,993048,991626,995802,993796,997483,992066,992742,997320,990650,997470,995513,996959,990182,997759,999710,998090,991814,992443,991229,991250,998808,996785,990910\n", + "Ground Truth: 991873,997913,991090,993834,999982,992822,999247,992096,992228,998500,993048,991626,995802,993796,997483,991616,992066,999165,992742,997320,990650,993616,997470,995513,996959,990182,997321,997759,990588,999710\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 106\n", + "Returned IDs: 997320,992944,990772,991498,996001,999710,990949,991131,996075,992041,993484,995336,998550,995372,992443,997305,992748,992119,994100,996085,993388,996318,994964,990585,998246,990957,993785,999374,997661,994415\n", + "Ground Truth: 997320,992944,990772,991498,996001,999710,990949,991131,996075,992041,993484,998365,995336,998550,995372,992443,997305,992748,992119,994100,992692,998232,996085,993388,996318,994964,994449,990585,998246,990957\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 107\n", + "Returned IDs: 995274,994964,999144,999026,997015,999437,998779,992630,990535,993289,998951,993168,996959,997918,994197,993492,991467,992216,995755,992384,999374,997002,993484,990958,995336,990957,997320,997976,992558,998797\n", + "Ground Truth: 994916,995274,994964,999144,999026,997015,999437,998779,992630,990535,993289,998951,993168,996959,997918,992047,997920,994197,993492,991467,999916,992216,995755,999596,992384,999374,997002,993484,990958,995336\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 108\n", + "Returned IDs: 994379,997319,996411,990223,995611,990675,995819,991810,997822,991943,995289,994382,993831,998156,995456,991350,992561,993561,995633,994783,998750,990889,995020,998765,990277,992582,994742,990516,995702,994283\n", + "Ground Truth: 994379,997319,991973,996411,990223,995611,990675,995819,999627,991810,997822,991943,995289,994382,993909,993831,998156,995456,991350,992561,997732,993561,995633,994783,998750,996607,990889,996906,995020,998765\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 109\n", + "Returned IDs: 999898,997028,996996,991311,994541,993695,997410,997920,993472,990974,997345,991923,998302,994783,999343,998169,998039,998122,993807,995384,996062,996219,990505,997878,991776,992120,996216,995895,997320,999876\n", + "Ground Truth: 999898,997028,996996,991311,994541,993695,997410,997920,993472,990974,997345,991339,991923,998302,994783,996939,999343,998169,998039,998122,990210,992931,993807,995384,999009,996062,996219,990505,995620,997878\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 110\n", + "Returned IDs: 997028,990557,996001,996219,999948,998039,991498,999058,998765,991930,994964,992446,993885,990752,994487,995102,991523,991776,991212,997320,997332,996318,992182,991184,995216,997920,995107,997410,999344,991994\n", + "Ground Truth: 997028,990557,996001,997381,996219,999948,994777,998039,991498,999058,998765,994061,995053,991930,995424,994964,992446,993885,990752,994487,995102,997876,991523,991776,991212,997320,997332,996318,992182,991184\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 111\n", + "Returned IDs: 996328,991882,990133,992450,990531,998067,995326,998610,997529,995295,997860,993939,994310,991769,990712,998710,990027,997103,994830,994742,996301,990435,993079,991885,996001,992703,996752,991585,993856,998671\n", + "Ground Truth: 996328,991882,990133,992450,990531,998067,995326,998610,997529,995295,997860,993939,994310,991769,990712,998710,990027,997103,994830,997933,994742,996301,990435,993079,991885,996001,992703,996752,996611,991585\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 112\n", + "Returned IDs: 995981,992470,992874,992384,995701,990862,996001,998721,993848,999904,998246,991045,990588,998445,995467,994879,998549,993342,999977,998921,995158,992167,997984,991459,997437,991245,996714,995336,995886,996075\n", + "Ground Truth: 993154,995981,992470,992874,992384,995701,995792,990862,996001,998721,993432,993848,999904,998246,991045,990588,998445,995467,994879,998549,993342,999977,998921,995158,997984,992167,991459,997437,991245,996714\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 113\n", + "Returned IDs: 996425,994819,994334,995453,997918,994804,994088,996046,997002,993465,996903,990995,995200,999277,995373,993407,995444,994446,999089,999595,990791,995928,998530,991942,997637,996864,992933,998259,990459,995401\n", + "Ground Truth: 996425,994819,994334,995453,997918,994804,994088,996046,997002,993465,996903,994362,999964,990995,995200,999277,995373,995444,993407,995374,994446,999089,999595,991603,990791,995928,998530,991942,997637,992260\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 114\n", + "Returned IDs: 991721,990096,996237,999444,996368,998362,995016,991154,990856,998039,995899,991869,994964,994925,999781,993168,990930,991958,997320,994961,998862,992994,990925,998950,992546,995701,993389,995678,995126,990811\n", + "Ground Truth: 991721,990636,990096,996237,999444,996368,998362,995016,991154,990856,998039,995899,991869,994964,994925,999781,993168,997053,990930,991958,997320,994961,998862,996661,992994,996677,997920,994140,990925,998950\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 115\n", + "Returned IDs: 995941,991782,996592,999089,993145,997096,995793,995336,990647,999200,990180,998572,991456,990275,990994,994111,999700,991379,995465,995886,999374,997748,994757,997371,997194,997479,997571,997022,996521,995453\n", + "Ground Truth: 995941,991782,996592,999089,993145,997096,995793,995336,990917,990647,999200,990180,998572,991456,990275,990994,994281,994111,999700,991379,995465,995886,999374,997748,990478,994757,997371,997194,997479,997571\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 116\n", + "Returned IDs: 998523,994922,992907,993355,996840,990043,993237,990334,990332,990448,995004,999453,999620,997918,990432,990925,991510,996196,990483,992612,996438,997496,998401,999076,998837,993143,996402,999676,996915,993570\n", + "Ground Truth: 998523,994922,992907,993355,996840,990043,993237,990334,997876,990332,990448,995004,999453,999620,997918,990432,990925,991510,996196,990538,990483,992612,996438,991609,997496,998401,995401,999076,998837,993143\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 117\n", + "Returned IDs: 996194,999601,998823,993014,994541,995695,990521,990169,995421,996396,994119,996429,998934,994280,995401,999496,994051,997316,990011,998250,992052,998968,991245,992313,996633,998817,992437,993335,996435,998067\n", + "Ground Truth: 996194,999601,998823,993014,994541,995695,990521,990169,995421,996396,994119,998348,996429,998934,994280,995401,999496,994051,998546,994817,997316,990011,998250,992052,998968,991245,992313,992131,996633,998817\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 118\n", + "Returned IDs: \n", + "Ground Truth: 995961,994850,994081,997470,998839,997320,998170,990949,991379,997301,994290,995374,990066,995422,998500,990910,994100,991743,993852,994757,995372,993646,998090,999647,998950,990532,995949,992228,996629,997479\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 119\n", + "Returned IDs: 995614,993426,990191,997820,996516,997292,990855,994415,998921,997483,998721,996387,999109,999271,994618,995623,996993,996771,999792,994233,992853,991131,995306,993484,993785,995119,998445,996135,997550,999036\n", + "Ground Truth: 995614,993426,990191,997873,997820,996516,997292,990855,994415,998921,997483,998721,996387,999109,999271,994618,995623,996296,996993,996771,999792,994233,992853,991131,995306,993484,993785,995119,998445,996135\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 120\n", + "Returned IDs: 996658,993288,992933,997391,997309,997918,996282,990956,997575,995080,995421,997897,996633,995862,991666,994455,996867,992231,995357,994409,991950,996878,997125,993023,995159,992052,994073,999303,999660,990521\n", + "Ground Truth: 996658,991864,993288,992933,997391,997309,997918,993094,996282,990956,997575,995080,995421,997897,996633,993211,995862,991303,991666,994455,996867,992231,995357,994409,991950,997462,992779,996878,994300,997125\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 121\n", + "Returned IDs: 996075,992147,999374,995515,991545,991743,992087,991889,994850,999074,996504,994776,997305,993355,990957,990083,994266,997192,995372,997320,996521,992880,991372,998921,997913,995596,994894,998779,999426,995443\n", + "Ground Truth: 996075,992147,999374,995515,991545,991743,992087,990118,991889,994850,999074,996504,994776,997305,991801,993355,990957,999140,990083,994266,998045,996648,997192,995372,993053,995374,997320,995192,993186,996521\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 122\n", + "Returned IDs: 998019,995801,990834,995203,993504,996233,990557,991648,996884,999526,990203,993179,994955,996001,992944,998764,998855,996908,997332,995366,995562,999320,995130,997002,996075,994829,995500,992041,999019,991037\n", + "Ground Truth: 998019,995801,990834,995203,993504,996233,990557,991648,996884,999526,990203,994081,993179,994955,996001,992944,998764,992627,998855,996908,997332,995366,995562,999320,995130,994449,997002,996075,998849,994829\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 123\n", + "Returned IDs: 999902,996326,994467,999383,997618,993931,992209,990364,991515,992104,993965,993275,996124,998051,995408,999079,995630,990998,991685,993609,992407,993583,995780,998536,998006,998855,990784,991303,996209,991755\n", + "Ground Truth: 999902,996326,994467,999383,997618,993931,992209,990364,991515,992104,993965,993275,996124,998051,995408,999079,995630,990998,991685,993609,992407,993583,995780,998536,998006,998855,990784,994770,991303,996209\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 124\n", + "Returned IDs: 995985,990595,998263,992384,998500,993934,996714,998921,997192,997984,995793,995824,990275,991379,995554,997842,992994,995181,992546,996727,999599,991829,999327,992228,991749,995927,995090,996504,995274,997889\n", + "Ground Truth: 995985,990595,998263,992384,998500,993934,996714,998921,997192,997984,995374,995793,995824,990275,991379,993072,995554,997842,992994,995181,992546,996727,999599,991829,999327,992228,991749,995927,995090,996504\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 125\n", + "Returned IDs: 990940,995269,994446,999932,993465,990242,995289,995444,993831,999000,995685,992160,995204,996867,997723,997046,997937,998070,999770,996302,994108,998779,993433,996323,999037,999277,997822,994571,996166,995507\n", + "Ground Truth: 990940,995269,994446,999932,993465,990242,995289,995444,993831,999000,992160,995685,995204,996867,997723,997046,997937,993412,998070,999770,996302,994108,998779,994405,993433,996323,999037,999277,997822,994571\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 126\n", + "Returned IDs: 997550,992635,997584,996075,996387,994618,995515,999426,998498,998921,997301,996516,995596,994971,993447,990697,999453,995372,996741,997984,994415,999526,997227,995033,997470,995119,993785,990855,997320,994233\n", + "Ground Truth: 997550,992635,997584,996075,996387,994618,995515,999426,998498,998921,997301,996516,995596,994971,993447,990697,999453,995250,995372,996741,997448,997984,994415,999526,997227,995033,997470,995119,993785,990855\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 127\n", + "Returned IDs: 998634,996001,997320,998246,993484,992839,991787,993053,994964,993280,991155,997332,999099,990585,990146,993168,993885,991498,999453,991721,991184,996219,990772,990096,993389,991732,991609,992896,990949,998365\n", + "Ground Truth: 998634,996001,997320,998246,993484,992839,991787,993053,994964,993280,991155,994961,997332,999099,990585,990146,993168,993885,991498,995678,999453,991721,991184,996219,990772,990096,993389,993154,990269,991732\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 128\n", + "Returned IDs: 997809,996546,991923,992718,998975,999380,993681,998331,991977,996163,990908,994398,993786,995138,993572,994852,992170,996158,995578,991157,999387,993975,996894,995737,992349,996585,999806,993839,992258,998853\n", + "Ground Truth: 997809,996546,991923,992718,998975,999380,993681,998331,991977,996163,990908,994398,993786,995138,993572,995986,994852,996658,992170,996158,995578,991157,998921,999387,993975,998026,996894,995737,992349,996585\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 129\n", + "Returned IDs: 992255,990650,996876,995794,990114,992822,991873,998947,999948,993083,991616,993048,999110,998434,990742,990066,995109,998540,990699,994461,991396,990311,990030,998899,993616,997539,991900,997320,994934,991593\n", + "Ground Truth: 992255,990650,996876,995794,990114,992964,992822,991873,998947,997451,999948,995485,997102,997948,998003,993083,997966,991616,993048,999110,995367,998434,990742,990066,995109,998540,990699,994461,991396,990311\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 130\n", + "Returned IDs: 993100,998710,995658,990735,992449,994998,994830,994858,999472,992796,993856,997305,997529,990771,998105,992006,994663,999354,994558,993939,994799,996603,993053,994185,999372,995200,991938,999192,999302,994131\n", + "Ground Truth: 993100,998710,995658,990735,992449,994998,994830,994858,999472,992796,993856,999391,997305,997529,990771,998105,990439,992006,994663,996147,990041,999354,994558,995873,993939,994799,998379,996603,993053,994822\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 131\n", + "Returned IDs: 998787,993366,994834,990952,995393,999373,998565,990167,994833,999555,996553,998959,996408,990358,995299,990173,995231,995835,993528,993560,994756,991313,994374,993014,998536,995168,994682,994150,993948,992862\n", + "Ground Truth: 998787,993366,994834,990952,995393,999373,998565,990167,994833,995326,999555,996553,998959,996408,990358,995299,990173,995231,992282,995835,993528,993560,994756,991313,994374,993014,998536,995168,994682,994150\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 132\n", + "Returned IDs: 999236,996716,992790,999820,998723,991011,993281,996371,996067,992179,993504,994946,998303,996062,998401,992576,993071,998045,994113,991857,993979,997331,994188,997002,994964,993492,995289,995589,996198,999898\n", + "Ground Truth: 999236,996716,992790,999820,998723,991011,993281,996371,996067,992179,993504,994946,998303,996062,998401,992576,995401,993071,998045,994113,991857,990294,993979,992367,997331,996220,994188,990244,997002,994964\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 133\n", + "Returned IDs: 999503,997790,993055,999840,994280,997926,993489,995203,992862,997897,992521,990399,990005,992958,998250,997918,998303,991661,994042,991630,994946,991332,998933,992747,996657,998697,996368,993260,996733,990350\n", + "Ground Truth: 999503,997790,993055,999840,994280,997926,993489,991397,995203,992862,992979,997897,992521,990399,990005,992958,998250,990802,997918,998303,991661,994042,994531,991630,994946,991332,998933,992747,996657,993143\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 134\n", + "Returned IDs: 991227,992558,992052,993478,998536,996531,995783,994280,995214,994972,995408,990956,991052,998104,998397,993121,992997,993747,996438,996658,995401,997897,993727,992521,996328,999179,994119,990776,999514,995184\n", + "Ground Truth: 991227,992558,992052,993787,993478,992933,998536,996918,996531,995783,994280,995214,994972,995408,990956,992318,993673,991052,998104,998397,993121,992997,993747,996438,996658,995401,996930,997897,993727,992521\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 135\n", + "Returned IDs: 994406,992767,998431,997391,999960,998779,997634,994474,994671,996727,996903,995374,995453,997012,995034,997157,997640,993412,990647,993289,996172,995893,995369,998529,994092,997192,997702,997462,994794,997995\n", + "Ground Truth: 994406,992767,997799,998431,995403,997391,999960,990337,998779,997634,993026,994474,998080,994671,996727,996903,995374,996047,995453,997012,995034,997157,997754,997640,993412,990647,993289,996172,995893,995607\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 136\n", + "Returned IDs: 991324,999740,992698,993725,990011,990484,991390,994830,995756,990711,992479,997142,990743,994751,998710,995002,991885,996358,996633,997714,992664,991911,995435,992679,992480,996001,997933,999377,996275,990735\n", + "Ground Truth: 991324,999740,992698,993725,990011,990484,991390,994830,995756,990711,992479,997142,990743,994751,998710,990089,995002,991885,999555,996358,999922,996633,997714,992664,991911,995435,992679,990439,992480,996001\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 137\n", + "Returned IDs: 995771,998542,993364,997543,994584,999263,991533,993250,991889,990515,997301,992401,999291,991558,998211,992147,993581,994939,999968,992818,998539,999270,996218,996504,990958,994670,999089,992384,999145,999982\n", + "Ground Truth: 995771,998542,993364,997543,994584,995034,999263,991533,993250,991889,990515,997301,992401,999291,991558,998211,992147,993581,994939,999968,992818,998539,999270,996218,996504,990958,994670,999089,992384,999145\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 138\n", + "Returned IDs: 999350,994864,994488,995167,996001,992169,996406,997679,991212,992182,998122,994510,991596,990557,993931,999130,999304,999623,994467,999099,994119,990434,997332,997395,993032,998725,996996,990364,996028,994280\n", + "Ground Truth: 999350,994864,994488,995167,996001,992169,996406,999179,997679,991212,999679,992182,996803,998122,994510,991596,993931,990557,999130,999304,999623,994467,999099,994119,990434,997332,996075,997395,993032,994416\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 139\n", + "Returned IDs: 994168,992469,999500,995289,990126,990930,996811,998913,994783,991154,990340,993747,994742,997864,998495,990523,993642,996116,999977,990399,995401,996609,996062,994325,993561,995633,994431,994140,994190,990012\n", + "Ground Truth: 994168,992469,999500,995289,990126,997723,993656,990930,999586,996811,998913,995562,994783,991154,990340,993747,993400,994742,996219,995095,994759,997864,998633,998495,997797,990523,998156,990266,993642,996116\n", + "Recall: 0.6000\n", + "\n", + "Query ID: 140\n", + "Returned IDs: 999867,990957,990949,998246,997320,994964,993484,990736,991379,993853,992107,995949,995928,997913,998500,991895,995942,997371,999982,996785,994192,998170,995336,994100,995884,993518,997470,998365,999374,990925\n", + "Ground Truth: 999867,990957,990949,998246,998294,997320,994964,993484,990736,991379,993853,992107,995949,995928,997913,998500,991895,995942,997371,999710,999982,996785,990397,994192,993072,996560,998163,998170,995336,991743\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 141\n", + "Returned IDs: 995004,995639,995788,997288,996402,994922,994785,998657,992641,998241,994612,995588,993514,992302,995817,996840,991151,993708,996100,991172,990300,991473,993355,997486,993282,991193,997220,990902,993143,991510\n", + "Ground Truth: 995004,995639,999888,995788,997288,996402,994922,994785,998657,992641,993675,998241,998973,994612,995588,993514,992302,995817,996840,991151,993708,996100,991172,990300,991473,993355,993282,997486,991193,997220\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 142\n", + "Returned IDs: 993250,990515,998211,997543,999823,993364,997301,993669,999263,991743,996218,992401,999968,999089,990561,990760,997320,997192,995886,993798,998921,999291,999145,992384,999374,996504,995928,992147,999026,990585\n", + "Ground Truth: 993250,990515,998211,997543,999823,993364,997301,993669,999263,991743,996218,992401,999968,999089,990561,990760,997320,998445,995034,997192,993648,995886,993798,998921,997228,999291,999145,992384,999374,996504\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 143\n", + "Returned IDs: 999342,990662,996147,998878,994223,994708,996739,990771,992636,997550,996657,990573,994523,993344,999130,994830,994742,996001,998165,998803,993848,999570,999302,995873,995743,998949,994185,998076,998097,997565\n", + "Ground Truth: 999342,990662,996147,998878,994223,994708,996739,990771,992636,997550,996657,993599,991255,990573,994523,993344,992293,999130,994830,994742,996001,998165,998803,993848,999570,995827,999302,995873,995743,998949\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 144\n", + "Returned IDs: \n", + "Ground Truth: 995515,992835,990155,998244,996419,993582,993028,994216,998837,990142,991149,999053,990313,993237,991999,997591,994612,994281,992020,990652,997403,990943,994475,994497,990827,997465,990043,992662,994898,996485\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 145\n", + "Returned IDs: 990897,990942,995516,993055,997806,998758,996728,997506,995237,995268,993580,998346,993948,992636,991630,990005,991161,998067,995159,999775,993044,992052,992521,993275,990956,994380,993560,995184,990593,998697\n", + "Ground Truth: 990897,990942,995516,993055,997806,998758,996728,997506,995237,995268,993580,998346,993948,998045,992636,991630,990005,991161,998067,995159,999775,993044,992862,992052,994891,992521,993275,990956,994380,993560\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 146\n", + "Returned IDs: 994577,992423,999806,991157,998260,993552,998568,992617,992309,995515,997305,998495,999122,998921,999079,999526,993260,995080,992521,995372,998607,991743,999317,992944,997320,997227,999908,990908,999367,990843\n", + "Ground Truth: 994577,992423,999806,991157,998260,993552,998568,992617,992309,995515,997305,998495,999122,998921,999079,999526,997301,993260,995080,992521,995372,998607,991743,999317,992944,997320,997227,996281,999908,990908\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 147\n", + "Returned IDs: 990132,991664,993390,992582,997411,994717,990361,994656,995253,998779,991415,992989,996379,999646,991836,996867,994382,995701,996368,999402,995268,991350,997058,994092,997209,996659,994793,998922,993566,996388\n", + "Ground Truth: 990132,991664,993390,992582,997411,992604,992293,994717,990361,994656,995253,998257,998779,991603,991415,992989,996379,999646,991836,996867,994382,995701,996368,999402,995268,991350,997058,994092,997209,997723\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 148\n", + "Returned IDs: 997379,998455,993052,999077,996193,998180,993260,993696,999210,999784,998950,992571,993781,994341,992309,993504,996305,995203,997370,995443,990580,992944,991348,991157,996075,995365,990043,999122,994552,992169\n", + "Ground Truth: 997379,998455,993052,999077,996193,998180,993260,993696,999210,999784,998950,992571,993781,994341,992309,995094,993504,996305,995203,997370,995443,990580,992944,991348,991157,990414,996075,995365,990043,999122\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 149\n", + "Returned IDs: 997209,999843,992582,996220,994747,993857,997309,991189,994742,992460,990328,993549,995633,990132,991334,994119,998410,997544,991664,994835,997645,991269,992989,998910,999279,998495,993209,991075,996809,990126\n", + "Ground Truth: 997209,999843,992582,996220,994747,993857,997309,991189,994742,992460,990328,993549,995633,993948,990132,991334,994119,998410,997544,991664,994835,997645,991269,992989,992604,997208,992309,998910,999279,998495\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 150\n", + "Returned IDs: 992678,996848,998067,997915,991044,990331,991747,992097,999980,992521,997675,992569,998045,995268,991103,990314,996368,996277,996505,998060,993256,996719,995963,992187,994861,999402,993313,994817,995815,992448\n", + "Ground Truth: 992678,996848,998067,997915,991044,990331,991747,992097,999980,992521,997675,992569,998045,995268,998021,991103,990314,996368,996277,996505,998060,993256,992367,996719,995963,992187,994861,999402,997723,993313\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 151\n", + "Returned IDs: 999578,992448,994409,992660,992801,997832,999043,996038,999530,998714,991699,997868,997406,995139,998045,998979,999508,996607,991027,995507,994554,993433,996769,991671,993561,998545,999940,993421,999884,996867\n", + "Ground Truth: 999578,992448,994409,993468,992660,992801,996618,997832,999043,996038,999530,998714,991699,997868,996813,997406,995139,998045,998979,999508,996607,991027,995507,994554,993433,996769,991671,993561,993280,998545\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 152\n", + "Returned IDs: 997281,998720,992225,996857,992330,997305,990866,998514,997811,992068,998568,996230,995536,996075,993834,995638,994636,993551,999651,991373,991256,995701,996576,999636,993139,990051,993809,994572,992167,994531\n", + "Ground Truth: 997281,998720,992225,996857,992330,997305,990866,998514,997811,992068,998568,996230,995536,996075,993834,999987,995638,994636,993551,990642,999651,991373,991256,995701,996576,999636,993139,990051,993809,994572\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 153\n", + "Returned IDs: 992239,995637,990264,990814,991981,991590,999924,999767,992713,994891,993080,997494,994968,991428,990516,991769,995012,994094,992318,997698,990364,990270,992437,996728,999942,990853,992827,994742,999399,996911\n", + "Ground Truth: 992239,995637,990264,990814,991981,991590,999924,999767,992713,994891,993080,997494,994968,991428,990516,991769,995012,994094,998999,992318,997698,990364,990270,992437,996728,998869,999942,990853,992827,996658\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 154\n", + "Returned IDs: 994168,990080,990266,998633,991943,996093,999586,994742,999587,998156,991981,993080,993717,999453,992169,991585,995295,990957,998980,996867,991185,994431,996809,998256,995873,990860,997059,995274,994140,999957\n", + "Ground Truth: 994168,990080,990266,994131,992655,998633,991943,996093,999586,991255,994742,997723,999587,998156,991981,993080,993717,992582,999453,992169,997360,991585,991334,995295,994129,990957,998980,996867,991185,994431\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 155\n", + "Returned IDs: 995252,999024,993241,998462,999358,992066,990585,997015,998951,990817,996959,997771,999144,993544,990231,990535,995824,993562,994595,999650,999535,997320,999531,995716,997314,996785,996871,997256,996196,998090\n", + "Ground Truth: 995252,999024,993241,998462,999358,992066,990585,997015,998951,990817,996959,997771,999144,993544,990231,990535,995824,993562,994595,999650,998160,999535,997320,999531,995716,997314,996785,991362,996871,997256\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 156\n", + "Returned IDs: 999267,993637,991128,993053,995283,995659,990089,990368,999338,990629,995345,991412,991992,993410,999806,992636,997691,997052,994386,995638,998302,995467,998021,999446,996368,998215,999791,993561,995689,991795\n", + "Ground Truth: 999267,993637,991128,993053,995283,995659,990089,990368,999338,995274,990629,996701,990133,995345,991412,991992,993410,999806,992636,997691,997052,994386,995638,998302,995467,998021,999343,990260,997340,999446\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 157\n", + "Returned IDs: 990593,995907,998671,999934,995083,998315,992450,993476,995268,998537,998055,996431,995191,992106,994142,998256,990041,996495,997652,991769,992859,997025,997806,993466,994891,996328,990069,999360,996055,997185\n", + "Ground Truth: 990593,995907,998671,999934,995083,998315,992450,993476,995268,998537,998055,996431,995191,992106,994313,994142,999441,998256,990041,996495,997652,991769,998072,992859,997025,997806,993466,994891,996328,990069\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 158\n", + "Returned IDs: 990890,996219,991928,998899,992931,991249,998540,994069,999068,999343,994964,992330,991593,996343,997913,994486,998550,999620,997305,990210,998837,996646,993264,994366,991999,990957,994783,997022,998464,999815\n", + "Ground Truth: 990890,996219,991928,998899,992931,991249,998540,994831,994069,999300,990545,999068,999343,995942,994964,992330,991593,991743,996343,997913,993282,990404,994486,998550,990666,999620,991006,995210,997305,990210\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 159\n", + "Returned IDs: 992094,994119,996245,997944,998150,994193,990226,994280,992703,992052,992437,990790,994968,997309,997897,993915,997056,998355,995408,996277,990784,991011,994375,994472,997881,998165,995017,993778,999643,992576\n", + "Ground Truth: 992094,994119,996245,997944,998150,994177,994193,990226,990527,994280,999461,992703,992052,998755,993037,992437,990790,994968,997309,997897,993483,993915,992652,997056,993288,995401,998355,995408,996277,990784\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 160\n", + "Returned IDs: 992604,993754,995807,995527,999662,999445,997781,990915,990540,995953,991051,990657,990729,998297,991204,991110,994187,997918,995373,994753,998858,991987,996043,997158,999306,995805,997675,995105,998697,997723\n", + "Ground Truth: 992604,993754,995807,995527,999662,999445,997781,990915,990540,995953,991051,990657,990729,998297,991204,991110,991571,994187,997918,997212,995373,994753,998858,991987,996043,997158,999306,995805,997391,997675\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 161\n", + "Returned IDs: 998294,996504,999765,997913,994266,998921,993518,991406,997984,990760,999263,993669,998500,995755,990925,994978,995090,998987,997192,990595,997371,998540,995793,990958,994415,994775,995949,993143,990560,995942\n", + "Ground Truth: 998294,996504,999765,990917,997913,994266,998921,993518,991406,997984,991895,990760,999263,993669,998500,995755,990925,994978,990829,995090,998987,990936,997192,990595,996560,997371,998540,995793,998344,999647\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 162\n", + "Returned IDs: 995110,993478,991103,996505,995032,998397,995783,999646,991747,995401,997675,995701,993727,990537,995521,991413,990956,993738,997918,997752,995184,995886,990872,992790,994280,990930,993948,998874,996715,991789\n", + "Ground Truth: 995110,993478,991103,996505,995032,998237,998397,995783,999646,991747,995401,997675,995701,993727,990537,995521,991413,991010,990956,992457,993738,991630,997918,997752,991052,995184,991054,994232,992260,998045\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 163\n", + "Returned IDs: 998914,990678,996556,996028,998355,997380,995528,999891,990226,991426,996190,993725,994765,995317,996505,996016,992322,994119,998325,999479,992480,995449,995456,998982,999516,991747,997649,990776,994012,994587\n", + "Ground Truth: 998914,990678,996556,996028,998355,997380,995528,999891,998630,990226,991426,996190,993725,992521,994765,995317,996505,996016,992488,996728,992322,994119,998325,999479,992480,993820,995449,995456,993336,998982\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 164\n", + "Returned IDs: 999830,991540,990487,992280,994541,992934,995273,994661,993976,997374,993583,996194,995436,994913,992052,996178,990908,993931,994488,991977,992718,995159,998417,998370,997329,993256,998118,990364,995551,990270\n", + "Ground Truth: 999830,991540,990487,992280,994541,992934,995273,997960,994661,992467,993976,997374,993583,996194,998135,995436,994913,992052,997056,996178,993931,990908,994488,991977,992718,995159,998417,998370,996685,997329\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 165\n", + "Returned IDs: 991308,995120,994955,992958,992823,990364,997897,996001,997320,995167,990557,999526,990314,997790,997368,996657,995250,990096,992825,993375,998593,994081,994972,990409,995672,997926,995855,995740,997977,994809\n", + "Ground Truth: 991308,996764,995120,994955,992958,992823,990364,997897,996001,997320,995167,990557,999526,990314,991657,991898,994531,997790,997368,996657,995250,990096,992825,993375,998593,994972,994081,990409,995672,997926\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 166\n", + "Returned IDs: 994380,996064,996001,990392,990887,991866,991585,992832,990127,997332,994925,990712,993561,997732,998470,999126,992703,998122,992450,991831,998067,998416,993637,996505,994541,998862,998055,991185,993180,992595\n", + "Ground Truth: 994380,996064,996001,993710,990392,990887,991866,991585,992832,990127,997332,993648,994925,990712,993561,997732,998470,999126,993412,992703,998122,992450,991831,998067,998416,993637,996505,994541,998862,998055\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 167\n", + "Returned IDs: 997666,996733,999612,992234,996386,993132,993867,992209,999368,995593,998671,998781,994181,992716,992673,998598,991724,996209,990806,999388,995385,997693,993148,990423,994731,996893,995796,994335,999079,997685\n", + "Ground Truth: 997666,996733,999612,992234,996386,993132,993867,992209,999368,990712,995593,997807,998671,998781,990552,990630,994181,992716,992673,998598,991724,997442,996209,990806,999388,995385,997693,993148,992984,990423\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 168\n", + "Returned IDs: \n", + "Ground Truth: 993561,995923,996867,990975,996232,996769,997554,994419,998564,996001,993725,997406,992576,994925,996042,991958,996360,993813,998045,995963,994409,995216,992862,995401,993412,995052,997046,996277,993461,995873\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 169\n", + "Returned IDs: 994336,990103,996803,993727,998123,999244,993185,996806,996918,993805,993165,996535,994051,992558,997644,999296,991453,998968,995175,991227,999673,999029,994909,994035,996825,993478,991610,991054,997480,999191\n", + "Ground Truth: 994336,990103,996803,993727,998123,999244,993185,996806,996918,993805,993165,996535,994051,992558,997644,999296,991453,998968,995175,991227,999673,999029,994909,998279,994035,999098,996825,993478,996861,991610\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 170\n", + "Returned IDs: 999430,995332,995107,991961,991458,998416,998950,990377,993499,995560,996216,993807,995895,997331,995750,993617,993079,992780,997004,993479,996944,999210,992567,998157,996370,991382,993391,998732,990039,997586\n", + "Ground Truth: 999430,995332,995107,991179,991961,991458,994897,998416,998950,990377,993499,995560,996216,993807,995895,997331,990091,995750,993617,993079,990769,992780,997004,993479,996944,999210,992567,993335,998157,996370\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 171\n", + "Returned IDs: 996219,994990,994964,993378,991075,996052,997616,996593,997028,999453,995096,993342,996001,998039,990448,992780,990366,994777,990091,993648,999344,998464,998495,999151,990146,990632,999791,995107,998593,999261\n", + "Ground Truth: 996219,994990,994964,993378,991075,996052,994843,997616,992047,996593,997028,999453,995096,993342,996001,998039,990448,992780,990366,997920,994777,994472,999352,992261,994922,990091,993648,998633,999437,999344\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 172\n", + "Returned IDs: 990016,998105,992201,993588,994145,993053,995097,990435,990497,990536,994307,998548,991075,990662,994830,995289,994925,996974,999460,998165,993831,990811,996001,995823,996411,998950,998495,994730,995906,997691\n", + "Ground Truth: 990016,998105,992201,993588,994145,993053,995097,990435,990497,996147,990536,994307,998548,991075,990662,994830,995289,994925,990771,996974,999460,998165,993831,998097,990811,996001,993412,995823,996411,998950\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 173\n", + "Returned IDs: 992235,993832,996219,995483,993687,994288,993931,990705,994774,990931,994742,998080,993621,995455,997395,992645,996001,990722,997126,994062,998495,990126,995401,995932,990041,999961,999139,999430,992888,995134\n", + "Ground Truth: 992235,993122,993832,996219,995483,993687,997068,994288,999916,997149,993931,990705,994774,993566,990931,997790,997540,994742,998080,999551,993621,995455,999360,997395,992645,996001,999647,990722,997126,994062\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 174\n", + "Returned IDs: 997391,991698,997218,997918,990215,992933,995282,993933,992428,998199,997158,998022,994406,992179,996247,992790,995886,997514,994819,993288,990916,994982,992604,991459,991413,996792,994170,994280,992516,999154\n", + "Ground Truth: 997391,991698,997218,997918,990215,997799,992933,995282,993933,992428,998199,997158,998022,994406,992179,996247,992790,995886,997514,994819,991792,993288,990916,994982,992604,997901,991459,991413,996792,994170\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 175\n", + "Returned IDs: 994894,990689,997192,998974,995274,999444,994266,990957,993186,997035,997437,998697,990862,995536,996699,992811,990231,992357,997320,990958,996785,998649,998170,990146,998779,998500,995569,994100,995655,995701\n", + "Ground Truth: 994894,990689,997192,998974,995274,999444,994266,990957,993186,990498,997035,998326,997437,998697,991630,990337,990862,995536,996699,992811,990231,992357,997320,990958,996785,998649,993412,990799,998170,990146\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 176\n", + "Returned IDs: 994360,996178,996429,999676,991650,996219,995826,991409,999806,997301,990096,996001,998607,995672,996397,998180,993431,997456,998495,993696,997550,998401,999426,992423,998302,997586,999210,998921,993470,994742\n", + "Ground Truth: 992443,995401,994360,996178,996429,999676,995559,991650,996219,995826,991409,991607,999806,997301,990096,996001,994244,998607,995672,993831,998365,996397,997906,998180,993431,997456,995873,998495,993696,999900\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 177\n", + "Returned IDs: 999011,996730,992179,998536,996245,992790,991698,998874,994301,993673,993478,997918,995401,991459,992933,995154,995886,992558,999820,996040,994170,990214,993037,994334,998778,990623,992428,990479,994280,997702\n", + "Ground Truth: 999011,996730,992179,998536,996245,992790,991698,992685,998874,994301,993673,993478,997918,995401,991459,992933,995154,995886,992558,999820,998529,996040,994170,990214,991052,993037,994334,998778,990623,992428\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 178\n", + "Returned IDs: 994254,997809,998686,990487,991977,992889,997407,991923,998046,990776,995691,994165,998243,994760,990343,996068,999766,998975,993839,990522,997914,998357,993572,994010,995965,995506,996895,992645,998118,994119\n", + "Ground Truth: 994254,997809,998686,990487,991977,992889,997407,991923,998046,990776,995691,994165,998243,994760,990343,996068,999766,998975,993839,990522,993683,997914,998357,993572,994010,995965,995506,999913,996895,992645\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 179\n", + "Returned IDs: 993280,999453,996810,992395,998734,996001,991951,995988,990142,991517,995216,999076,991840,991248,999019,997320,998605,991999,992944,998697,995705,993018,997002,994280,996064,994081,993324,990652,994964,998416\n", + "Ground Truth: 990155,993280,999453,996810,992395,998734,990772,996001,991375,991951,995988,990142,994536,996915,996732,997876,991517,998567,995216,990778,995130,999076,991840,998523,991255,991248,999019,997320,999723,993154\n", + "Recall: 0.5333\n", + "\n", + "Query ID: 180\n", + "Returned IDs: 993324,999796,998104,999514,996918,990956,997420,995130,995401,992944,995332,993121,991961,995766,993347,994119,995438,998536,998180,991723,999881,995107,992052,994743,998950,995289,994541,996855,990776,999179\n", + "Ground Truth: 993324,999796,998104,999514,996918,990956,997420,995130,995401,992944,995332,993121,991961,995766,993347,994119,995438,998536,998180,991723,999881,995107,992052,994743,998950,995289,994541,996855,990776,993079\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 181\n", + "Returned IDs: \n", + "Ground Truth: 995895,996706,997442,998593,993805,993391,995438,992726,990844,998993,990133,999430,994431,994351,994742,996809,999435,999179,996930,998934,999587,998989,998990,994830,993335,997723,992864,997331,998055,994382\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 182\n", + "Returned IDs: 995289,995633,994742,995702,991585,997723,995200,990361,991185,993209,991664,996042,998162,995766,997182,998999,993831,996867,997822,996302,995648,992582,990940,992448,998165,997411,995401,996388,999862,995216\n", + "Ground Truth: 995289,995633,994742,995702,991585,997723,995200,990361,991185,993209,991664,996042,998162,995766,997182,998999,993831,996867,997822,996302,995940,999587,995648,992582,990940,992448,998165,997411,995401,996388\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 183\n", + "Returned IDs: 992002,991507,998162,995493,996607,998910,999054,997466,992448,990163,995458,996987,993046,990295,991671,991585,997529,994013,994865,998948,990924,993883,990804,996042,996815,995216,995766,993561,992801,992528\n", + "Ground Truth: 992002,991507,998162,995493,996607,998910,999054,997466,992448,990163,995458,996987,993046,990295,991671,991585,997529,992260,994013,994865,998948,990924,993883,990804,996042,996815,991603,998283,999475,995216\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 184\n", + "Returned IDs: 996689,990328,990600,994255,992445,999025,995456,994689,992052,995295,991277,996028,997329,991451,997309,994280,993560,997420,995166,990532,998589,995823,995273,997897,998182,995203,994946,998623,996580,997251\n", + "Ground Truth: 996689,990328,990600,994255,992445,999025,995456,994689,992052,995295,991277,992068,996028,997329,996110,991451,995832,997309,994280,998959,993560,997420,995166,990532,998589,995823,995273,997897,998182,995203\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 185\n", + "Returned IDs: 990193,996911,996277,993738,991769,995047,997059,999343,997554,991466,991349,991981,991185,995216,995184,992162,999402,998060,996719,992448,999731,991163,992521,997168,994140,991111,992526,991585,995052,998910\n", + "Ground Truth: 990193,996911,996277,993738,991769,995047,997059,999343,997554,991466,991349,991981,991185,995216,995184,992162,992879,994353,999402,998060,996719,991737,992448,999731,991163,992521,997168,990364,994140,991111\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 186\n", + "Returned IDs: 999858,994682,994233,994830,996183,997073,991885,998076,992372,997340,993101,993588,995299,993335,993917,998063,990582,992318,991409,990373,997109,991390,994269,990700,995735,993568,992026,996357,990392,992415\n", + "Ground Truth: 999858,994682,994233,994830,996183,997982,997073,991885,998076,992372,997340,993101,993588,993394,995299,993335,993917,998063,990582,992318,991409,990373,997109,991390,994269,990700,995735,993568,992026,996357\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 187\n", + "Returned IDs: 997391,997918,994334,990215,996282,994819,993492,995886,996903,997702,992979,992179,999820,995401,992211,993512,994088,994319,992428,999144,992933,994906,998892,990561,994670,996658,991698,992558,999804,993289\n", + "Ground Truth: 997391,997918,994334,990215,996282,994819,993492,995886,996903,999562,997702,992979,992047,992179,999820,995401,992211,993512,997782,994088,994319,992428,999144,994916,992933,994906,998892,990561,998011,994670\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 188\n", + "Returned IDs: 997106,992190,998409,995256,990675,992804,994852,991958,991923,999639,997864,994043,993342,991208,995701,996368,993283,992167,991530,997407,990622,998079,997739,995898,991350,996894,996127,997577,999216,992976\n", + "Ground Truth: 997106,992190,998409,995256,990675,992804,993791,998974,994852,991958,991923,999639,997864,994043,993342,991208,995701,996368,993283,992167,994796,991530,997407,990622,998079,995138,997739,995898,991350,994105\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 189\n", + "Returned IDs: 996062,992182,990096,992576,991857,996001,996277,998045,995130,991011,993561,995536,994964,995834,997586,991685,993433,993873,996884,994955,999740,996371,998039,991789,993906,998158,995289,996219,995052,998495\n", + "Ground Truth: 996062,992182,990096,992576,991857,997920,996001,996277,998045,995130,991011,993561,995536,994964,995834,993778,997586,991685,993433,993873,997320,996884,994955,999740,996371,998039,994419,991789,993906,998158\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 190\n", + "Returned IDs: 994105,994906,996050,992664,998855,991332,993866,995168,997332,999653,991038,997666,998623,995536,996075,995695,997906,996657,991691,999827,996001,999212,999543,999139,999724,990998,991630,996326,994280,995603\n", + "Ground Truth: 994105,994906,996050,992664,998855,991654,991332,993866,995168,998051,997332,999653,991038,997666,998623,995536,996075,995695,997906,996657,991691,999827,996001,999212,999543,999139,999724,990998,991630,996326\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 192\n", + "Returned IDs: 991372,993696,990628,994722,990885,996809,991109,992635,996452,990697,991172,994922,995210,995530,995596,998303,999337,996350,993504,991425,990627,996996,994964,993454,999210,993617,990957,999750,992041,991157\n", + "Ground Truth: 991372,993696,990628,994722,990885,996809,996296,997166,991109,992047,992635,997098,996452,990697,991172,998734,994922,995210,999219,992927,995530,994859,996875,995596,996913,998303,999337,996350,998790,993504\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 193\n", + "Returned IDs: 991515,990255,992015,995401,999595,998593,996930,990573,998687,998874,994119,993992,990412,996142,992367,991835,996657,996043,996759,999629,992104,995421,993275,996388,998990,995864,996001,990314,994743,998878\n", + "Ground Truth: 991515,993599,990255,992015,995401,999595,998593,997723,996930,990573,998687,998874,994119,992977,993992,990412,996142,992367,991835,996657,996043,996759,999629,992104,999669,995421,993275,996388,998990,995864\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 194\n", + "Returned IDs: 997319,995633,994283,991810,998750,995611,992561,994204,990223,997141,996411,991330,993261,995289,995401,995948,994148,992289,997155,994783,998765,994742,997117,990814,995190,998525,993747,999235,995274,994422\n", + "Ground Truth: 997319,995633,994283,991810,998750,995611,992561,994204,990223,997141,996411,991330,993261,995289,995401,995948,994148,992289,997155,994783,998765,994742,997117,990814,995190,998525,999627,993747,999235,995274\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 195\n", + "Returned IDs: 991890,996809,998786,999430,990361,996830,996284,998695,996930,998593,995461,994382,990255,994431,999009,990930,991846,995701,995401,990288,990167,995289,998874,995895,999994,995421,990941,993232,997630,991255\n", + "Ground Truth: 991890,996809,998786,999430,990361,996830,996284,998695,996930,998593,995461,994382,990255,996049,994431,999009,990930,991846,997723,995701,995401,990288,990167,995289,991025,998874,995895,999994,995421,990941\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 196\n", + "Returned IDs: 998697,992293,991530,993948,997884,995701,992862,992609,998758,999653,993738,995672,996828,995184,998831,990005,996368,991836,995536,996714,991844,993560,995516,997868,997928,990211,991630,998017,999639,993515\n", + "Ground Truth: 998697,992293,991530,993948,997884,995701,992862,992609,998758,999653,993738,995672,996828,995184,998831,990005,996368,991836,994125,995536,996714,991844,996657,998200,993560,995516,997868,997928,990211,991630\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 197\n", + "Returned IDs: 994111,999026,999144,997015,995274,997976,995336,998692,998365,994100,994964,992630,997320,997192,996959,991467,993561,995132,999374,995701,996714,993484,997371,995928,995502,998779,995204,997918,997571,998302\n", + "Ground Truth: 994111,999026,999144,997015,995274,997976,995336,998692,995374,995401,998365,994100,994964,992630,997320,992047,993412,997192,996959,991467,993561,995132,999374,992331,995701,998011,996714,993484,997371,994959\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 198\n", + "Returned IDs: 995645,991392,997291,998244,992612,990615,997227,990043,993143,991609,998837,999257,999076,998607,991999,992671,997496,999122,995515,990310,992146,996840,993237,997370,995203,990142,991013,998567,999784,991157\n", + "Ground Truth: 995645,991392,997291,998244,992612,992734,990615,997227,990043,993143,992336,991609,998837,999257,999076,998607,991999,992671,997496,999122,995515,990310,992146,996840,993237,997370,995203,990142,991013,996930\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 199\n", + "Returned IDs: \n", + "Ground Truth: 990382,997476,991581,999827,996062,992314,993981,994510,998095,994463,998436,992161,993289,990899,990520,991930,997790,996913,997642,993931,998158,992169,995954,997453,991596,995636,990557,990212,995185,991644\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 200\n", + "Returned IDs: 997320,997370,992944,997305,998950,991498,995886,993018,994266,995372,992041,990949,999453,995203,997301,995552,995927,993154,991801,999210,993260,990532,997470,995438,997913,993241,995841,997371,999701,998180\n", + "Ground Truth: 997320,997370,996811,992944,997305,998950,991498,995886,993018,994266,995372,992041,990949,995250,990772,999453,995203,997301,995552,990917,995927,993154,991801,999210,993260,990532,997470,995438,997913,993241\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 201\n", + "Returned IDs: 999082,991904,993879,997675,993385,997997,992801,991388,997588,999796,990875,994002,994413,996918,995159,990784,996505,995521,996386,993899,996472,998049,995080,994677,998405,998536,994835,992285,997459,993931\n", + "Ground Truth: 999082,991904,993879,997675,991061,993385,997997,992801,991388,997588,999796,990875,994002,994413,996918,995159,995200,990784,996505,995521,996386,993899,998107,996472,998049,995080,994677,997949,998405,998536\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 202\n", + "Returned IDs: 999267,998097,998021,990368,998076,993725,993715,996328,994016,998921,998812,995671,998848,999526,993719,991390,992825,999516,992068,990438,992636,990889,993053,990011,998536,992450,995705,994280,999338,995295\n", + "Ground Truth: 999267,998097,998021,990368,998076,993725,993715,996328,994016,998921,998812,995671,998848,999526,993719,991390,996429,992825,999516,990438,992068,992636,990889,993053,990011,998536,992450,995705,994280,999338\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 203\n", + "Returned IDs: 992927,993260,994988,994964,994537,996193,990786,997320,993055,992069,998568,994984,997918,993783,991795,992571,995536,992497,994789,992225,994552,999800,993948,993696,993834,992330,997909,999077,998790,999129\n", + "Ground Truth: 992927,993260,994988,990429,994964,994537,996193,990786,996053,997320,999216,996576,999362,993055,990885,999176,992069,998568,994984,997918,993783,991795,991675,992571,995536,992497,994789,992225,999711,994552\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 204\n", + "Returned IDs: 990487,994117,996391,998370,995896,991350,992929,997554,990211,997868,994793,994207,996368,996379,994809,996654,996657,990314,998990,996127,995873,993831,991585,992423,999639,990081,994032,996147,990662,993681\n", + "Ground Truth: 990487,994117,996391,998370,995896,991350,992929,997554,999587,990211,997868,994793,994207,996368,996379,993998,994809,996654,996657,997723,990314,998990,996127,991973,995873,998686,996181,993831,991585,992423\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 205\n", + "Returned IDs: 993637,997933,993906,992636,997052,995295,990392,993715,998039,998021,998097,990089,999138,990930,998593,997453,990845,991144,996001,993917,998076,990368,998440,995467,991021,997170,990027,999052,996906,990438\n", + "Ground Truth: 993637,997933,993906,993719,992636,997052,995295,990392,993715,998039,998021,994386,998097,990089,999138,990930,998593,997453,990845,991144,996147,996001,992547,993917,998076,990368,998440,995401,995467,991021\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 206\n", + "Returned IDs: 993885,993238,999533,997395,998029,995928,991743,997320,994266,992922,996805,991498,999827,994280,997732,998302,993250,997301,991932,993484,995961,997002,992182,994964,993080,996328,997822,990560,995861,990210\n", + "Ground Truth: 993885,993238,999533,997395,998029,995928,991743,997320,994266,992922,996805,991498,990814,999827,994280,997732,991611,993860,999290,998302,993250,997301,991932,993484,995961,991829,997002,992182,994964,993080\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 207\n", + "Returned IDs: 998165,992160,999782,995439,998045,996482,997406,992125,996769,992115,995289,996125,993830,994057,995052,991463,995182,990681,997046,990984,991603,993844,992925,995497,993390,998520,995285,992576,996277,991415\n", + "Ground Truth: 998165,992160,999782,995439,998045,996482,997406,992125,993236,996769,992115,995289,996125,993830,994057,995052,991463,995182,990681,997046,990984,991603,993844,992925,995497,993390,998520,996112,995285,992576\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 208\n", + "Returned IDs: 994273,999415,993127,995122,992449,995423,995702,994013,995701,999054,991636,996867,995182,995685,990082,995905,994954,990152,994409,998776,995401,997975,998518,997674,996232,992660,993856,995891,993232,998162\n", + "Ground Truth: 994273,999415,993127,995122,992449,995423,995702,994013,999745,995701,999054,991636,996867,995182,996897,995685,998636,990082,995905,997077,994954,990152,994409,998776,995401,997975,998518,997674,995303,996232\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 209\n", + "Returned IDs: 995167,996562,992199,993391,996216,997897,999748,998335,999829,990627,991037,993875,991596,999435,999093,992137,995626,992247,995214,997944,992151,990255,991148,997237,996501,993805,992825,993132,992056,992771\n", + "Ground Truth: 998731,995167,996562,992199,993391,996216,997897,999748,998335,999829,996986,990627,991037,993875,993195,991596,999435,993124,991170,999093,992137,995626,992247,999455,995214,997944,992151,990255,991148,997237\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 210\n", + "Returned IDs: 998749,992124,992213,992167,996576,992225,999041,991984,998232,993834,992238,996089,990550,996744,995516,996230,991157,995701,995897,994572,993766,990051,998950,990453,996138,993342,994809,996500,993902,999806\n", + "Ground Truth: 998749,992124,992213,992167,996576,992225,999041,991984,998232,993834,992238,990334,996089,990550,996744,995516,992276,996230,991157,995701,995897,994572,993766,998950,990051,990453,996138,993342,994809,996500\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 211\n", + "Returned IDs: 999453,994167,991987,991811,995596,996001,991459,999219,995899,990949,996657,990957,992979,996368,995281,997320,997258,992562,991835,993848,995866,999639,996744,992384,991392,994253,995455,998720,999586,998790\n", + "Ground Truth: 999453,994167,991987,990999,998567,991811,996161,991894,995596,996001,991459,993154,999219,995899,990949,996657,990957,991445,995855,992979,996368,995281,990542,997320,997258,992562,991835,993848,995866,999639\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 212\n", + "Returned IDs: 992824,995459,990894,996380,999639,991272,995631,995503,996742,990273,991432,996663,999908,998061,999387,997547,996894,991817,995728,993055,999674,993839,994560,993225,999346,998720,990061,992306,995724,999806\n", + "Ground Truth: 992824,995459,990894,996380,999639,991272,995631,995503,991064,996742,990273,991432,996663,999908,998061,999387,997547,996894,991817,995728,993055,999674,993839,994560,993225,999346,992752,998720,990061,992306\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 213\n", + "Returned IDs: 993879,995080,997675,995521,992801,996198,990244,999082,994170,994409,996505,993030,996277,998045,993561,992448,997822,998049,997832,996472,997020,993385,993738,994113,995648,995032,994589,996042,997723,993313\n", + "Ground Truth: 993879,995080,997675,995521,992801,991904,996198,990244,999082,994170,994409,996505,995401,993030,990527,996277,998045,993561,992448,995873,997822,998049,997832,991603,996472,994677,997020,997766,992260,990140\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 214\n", + "Returned IDs: 994280,991747,990434,997897,995184,993280,993931,993533,993738,990364,998067,996062,995047,990802,995876,992636,993324,993787,991255,990956,999827,994891,990951,998416,990993,998697,997679,996328,990784,997790\n", + "Ground Truth: 991860,994280,991747,990434,997897,990212,995184,998006,993280,993931,993533,993738,990364,998067,996062,995047,990802,995876,995281,992636,993324,993787,991255,990956,999827,994891,990951,998416,994389,990993\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 215\n", + "Returned IDs: 996062,998122,998495,999898,990194,990096,997242,990434,999827,991893,993106,999555,996219,996001,996028,998861,995655,995536,992056,994266,995562,999139,990167,990382,990796,998593,991296,991037,997586,996668\n", + "Ground Truth: 996062,998122,998495,999898,994869,990091,990194,990096,997242,990434,999827,991893,993106,999555,996219,996001,996028,998861,995655,995536,992056,994266,995562,999139,990167,990382,990796,998593,994200,997790\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 216\n", + "Returned IDs: 997429,995184,993238,991856,990958,992357,993648,999533,995963,994266,996861,993168,995800,998458,990925,995927,997852,995783,993885,999374,992147,995884,997309,992637,993014,998495,992994,995755,997803,995771\n", + "Ground Truth: 997429,995184,993238,991856,990958,992357,993648,994835,999533,995963,994266,995034,996861,991714,992008,993168,995800,994956,998458,990925,995927,997852,995783,993885,992526,999374,992147,995884,990399,997309\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 218\n", + "Returned IDs: 990711,990392,997529,991703,993053,998862,993725,996411,994730,995002,992156,993240,998913,999302,999377,995633,994742,990701,998495,998779,994830,990435,993166,993264,992680,996408,993856,995601,992024,998710\n", + "Ground Truth: 990711,990392,997529,991703,993053,998862,993725,996411,994730,995002,992156,993240,998913,999302,999377,995633,994742,990701,998495,998779,994830,995401,990435,993166,993264,995096,992680,996408,993856,991390\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 219\n", + "Returned IDs: 999343,993412,990366,996219,998779,995274,992630,998210,994964,998862,995336,990958,997479,998302,990974,998458,994100,999144,999026,996727,997320,990934,998090,990271,992384,996714,999647,990866,994990,998246\n", + "Ground Truth: 999343,993412,990366,996219,993909,998779,995274,992630,998210,994964,998862,995336,990958,990666,997920,997479,998302,990974,995374,990171,992008,998458,999331,994100,991311,999144,991362,999026,996727,997320\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 220\n", + "Returned IDs: 994643,998536,997752,993232,999117,998383,999155,990551,993995,996809,992288,998695,996797,991532,992558,997666,995963,990941,994063,991571,996043,993077,997131,994742,992604,992809,996395,990361,990991,995274\n", + "Ground Truth: 994643,998536,997752,993232,999117,998383,999155,990551,995625,993995,996809,992288,998695,996797,991532,992558,991630,997666,995963,990941,994063,991571,996043,993077,997131,991675,994742,995401,992604,992809\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 221\n", + "Returned IDs: 995596,995515,993143,991840,991848,999122,999784,999806,996840,990432,994709,997403,993748,995443,996219,999676,992979,999144,991395,991485,997135,997379,999613,992309,996062,997918,993237,994577,999453,994922\n", + "Ground Truth: 995596,995515,993143,991840,991848,999122,999784,999806,996840,990432,994709,997403,993748,995443,996219,995365,999676,992979,999144,991395,991485,992047,997135,997379,999613,991977,992309,996062,997918,994916\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 223\n", + "Returned IDs: 997030,990096,994717,994643,994413,993666,992638,993115,999530,998257,991309,994454,999898,991190,998277,999555,997211,993566,992369,995218,994677,998045,997568,990361,998536,994835,992161,994656,991937,990132\n", + "Ground Truth: 997030,990096,994717,994643,994413,993666,992638,993115,999530,998257,991309,994454,999898,990815,997077,997154,991190,997871,998277,999555,997211,993566,992369,995218,994677,998045,991596,991188,997568,990361\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 224\n", + "Returned IDs: 990049,995609,996194,999528,995043,995551,994326,993678,991241,996232,990417,993181,990011,996435,992566,994541,992068,999830,992473,993245,994317,995475,995537,993315,997906,996867,992563,995317,992951,995666\n", + "Ground Truth: 990049,995609,996194,999528,995043,995551,994326,993678,991241,996232,990417,993181,990011,996435,992566,994541,992068,999830,998769,992473,990121,993245,994317,995475,995537,993315,997906,990153,996867,992563\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 225\n", + "Returned IDs: 997806,991161,997251,993580,994280,994365,998067,994380,990294,991904,990942,991066,996960,999774,991103,996918,999294,992992,995268,994142,997918,993044,996328,991610,996683,994639,999994,997601,991038,993948\n", + "Ground Truth: 997806,991161,997251,993580,991273,994280,994365,998067,994380,990882,990294,991904,990942,991066,996960,992487,993439,999774,991103,996918,999294,992992,995268,994142,993727,997918,993044,996328,991610,990875\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 226\n", + "Returned IDs: 991140,999000,997723,996038,995685,998779,998999,995182,999412,991603,996968,998165,997832,997020,995648,994409,998297,996867,994948,996734,993103,990940,995507,995289,995421,997822,995401,998593,995052,996432\n", + "Ground Truth: 991140,999000,997723,996038,995685,998779,998999,995994,995182,999412,991603,996968,998165,997832,997020,995648,994409,998297,996867,994948,996734,993103,990940,995507,995289,997042,990406,995421,997822,995401\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 227\n", + "Returned IDs: 992511,998974,996252,997437,993457,994894,995838,997174,996917,997728,999599,993978,995755,998649,990957,997320,996192,995274,997304,994630,992531,993049,992296,997035,996700,997371,999333,998460,998090,999074\n", + "Ground Truth: 992511,998974,996252,997437,993457,990337,999867,994894,995838,997174,996917,997728,993222,999599,993978,995755,998649,994916,990957,997429,997320,996648,996192,995616,995274,995963,997304,994630,995374,991995\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 228\n", + "Returned IDs: 998060,993487,994203,992352,994106,992098,990573,992000,991057,996075,994488,996001,995079,995544,991212,994013,990662,990314,999172,997998,998593,996898,995873,994954,992569,995640,992097,991394,999590,997554\n", + "Ground Truth: 998060,993487,994203,992352,994106,992098,990573,992000,991057,996075,994488,996001,995079,995544,991212,994013,990662,990314,999172,996043,995515,997998,998593,996898,995873,994954,992569,995640,992097,991394\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 229\n", + "Returned IDs: 992558,990279,997547,997124,996196,990538,999715,995336,995886,996606,993948,994334,997859,992718,991157,996609,996918,999860,992857,995552,996721,994163,993363,990343,991498,994280,998697,991272,996318,992173\n", + "Ground Truth: 992558,990279,997547,997124,996196,990538,999715,995336,999908,995886,996606,993948,994334,997859,992718,990989,992971,991157,991052,996609,996918,999860,992857,990930,995552,996721,994163,993363,990343,991498\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 230\n", + "Returned IDs: \n", + "Ground Truth: 994266,995184,997320,997913,990231,998170,992546,991256,999647,992330,995701,993853,999041,997305,995417,996296,992637,996075,997554,999761,990958,996661,993834,997449,990650,996083,993168,990535,992748,991045\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 231\n", + "Returned IDs: 998913,994783,998823,998370,998196,995551,991428,994742,992566,991981,994806,994692,995289,995397,995348,992415,995948,995030,992827,997630,996411,991330,995633,998543,994110,996609,994587,997529,990126,997823\n", + "Ground Truth: 998913,994783,998823,998370,998196,995551,991428,994742,992566,991981,994806,994692,995289,995397,990846,995348,992415,995948,993939,995030,992827,997630,996411,991330,995633,998543,995300,994110,998028,996609\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 232\n", + "Returned IDs: 995841,997301,990405,995264,994942,999526,997292,990546,990191,992451,995372,991498,999523,998921,995623,996135,997470,996516,997320,998445,995306,993484,997333,999904,996771,993364,999338,998721,999594,999281\n", + "Ground Truth: 995841,997301,990405,995264,994942,999526,997292,990546,990191,992451,995372,991498,999523,998921,995623,996135,997470,996516,997320,998445,995306,993484,994741,997333,999904,996771,993387,993364,997876,999338\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 233\n", + "Returned IDs: 992981,992445,997806,996328,994336,990956,996728,996861,994125,993055,990005,997420,997884,996211,996918,997506,994142,999775,993948,996189,998959,993560,990961,993877,998495,998067,995516,994280,990226,994380\n", + "Ground Truth: 992981,992445,997806,996328,994336,990956,996728,996861,998017,992979,994125,993055,990005,997420,997884,996211,996918,997506,994142,999775,993948,996189,998959,993560,992293,990961,993877,998495,998067,995516\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 234\n", + "Returned IDs: 990067,994627,990092,994776,998550,992396,992147,992134,990470,996219,991015,991157,992942,999898,998899,996118,998090,990595,991014,997371,999144,995942,999784,998401,990231,998921,994964,990090,994757,993260\n", + "Ground Truth: 990067,994627,990092,994776,998550,992396,992147,992134,993412,990470,996219,998380,993633,991015,991157,992942,993852,999898,998899,996118,998090,990595,991014,997371,999144,995942,994831,999784,999102,998401\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 235\n", + "Returned IDs: 990435,994830,994742,995633,998913,995200,996132,993053,997529,992424,990012,996001,990621,999352,996219,995097,995289,994806,992446,999192,995401,991588,998495,990701,993335,998221,994307,996411,995002,993166\n", + "Ground Truth: 990435,994830,994742,995633,998913,995200,996132,993053,997529,992424,990012,996001,990621,999352,999261,996219,995097,995289,994806,992446,999367,999192,994145,995401,991588,998495,990701,993335,998221,994307\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 236\n", + "Returned IDs: \n", + "Ground Truth: 998971,991596,992673,999663,999139,994510,996406,997127,999898,996985,996706,996809,999555,996733,991420,997980,998149,999435,992365,998006,996209,994335,992234,997242,999498,990226,995104,997506,991589,999783\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 237\n", + "Returned IDs: 999093,991425,992571,991037,998303,993504,995839,994134,999840,990885,998447,990096,992168,994221,997977,990409,992351,992747,998154,997897,993260,992082,998495,995203,998732,999750,990212,991332,998299,991491\n", + "Ground Truth: 999093,991425,992571,991037,998303,993504,994468,995839,994134,999840,990885,998447,990096,992168,994221,997977,990409,992351,992747,998154,997897,993260,991596,992082,998495,998732,995203,999750,995167,990212\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 238\n", + "Returned IDs: 995289,998990,996930,995401,997822,997723,999587,993831,998874,993673,998165,996867,994343,997406,996379,995633,995190,998593,995685,992703,993412,995702,997046,997630,990517,992685,993848,997832,991603,997662\n", + "Ground Truth: 995289,998990,996930,995401,997822,997723,999587,993831,998874,993673,998165,996867,994343,997406,996379,995633,995190,998593,995685,996607,992703,993412,995702,997046,998323,996294,997630,990517,992685,990930\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 239\n", + "Returned IDs: 999676,998921,991485,995515,992612,994922,995788,998401,996840,999806,998950,999077,996351,991833,992944,996915,995983,998244,993260,998697,991157,990096,993143,999698,993289,993781,997320,995080,994195,995372\n", + "Ground Truth: 999676,995693,998033,998921,991485,995515,992612,994922,995788,998401,996840,999806,998950,999077,999261,995401,993470,996351,991833,992944,996915,995983,998244,993260,998697,991157,990096,996371,993143,999698\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 240\n", + "Returned IDs: 997397,999110,999523,997301,999471,992451,995623,999904,991916,993484,990146,995306,990546,998760,997470,990405,997820,993426,993083,997292,995264,997320,992066,999242,995045,990191,996516,998921,995454,991134\n", + "Ground Truth: 997397,999110,999523,997301,999471,992451,995623,999904,991916,993484,990146,995306,990546,998760,997470,990405,997820,993426,993083,997292,995264,997320,992066,999242,995045,999247,991582,990182,990191,996516\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 241\n", + "Returned IDs: 998697,991530,999639,995672,997586,995562,997506,991485,991256,998303,999783,995894,993055,996918,990399,996001,999806,995418,995455,993342,995536,999216,996406,997258,997320,990930,999041,994906,995701,993834\n", + "Ground Truth: 998697,991530,999639,995672,997586,995562,997506,991485,991256,998303,999783,997531,995894,993055,995855,996918,990399,996001,999806,995418,995455,990453,993342,995536,999216,995250,996406,997258,997320,990930\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 242\n", + "Returned IDs: 995659,999271,990585,997668,997286,990089,999338,990000,994942,994618,997292,997984,995841,997470,997301,996516,995306,992069,991503,997320,998721,998921,995372,998923,998445,993419,994964,992451,997113,997820\n", + "Ground Truth: 995659,999271,990585,997668,997286,990089,999338,990000,994942,994618,997292,997984,995841,997470,997301,996516,995306,992069,991503,997320,998721,998921,995372,991570,998923,998445,998461,993419,994964,992451\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 243\n", + "Returned IDs: 990467,994688,998649,996192,996869,990397,999301,998458,991268,995274,994630,990925,995942,996196,999500,996219,995502,997079,992630,997913,992291,998500,998797,991096,990949,996252,994783,998302,996727,992994\n", + "Ground Truth: 990467,994688,998649,996192,996869,990397,993656,999301,998458,991268,995274,992273,990315,993725,993412,991995,994630,996411,990925,995942,996196,999500,996219,995502,997079,992630,997913,992291,998500,998797\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 244\n", + "Returned IDs: 993158,990620,996389,998647,995410,995511,999715,996609,998827,992215,990731,995271,999240,997893,993473,997195,994334,993690,997859,992971,996476,990249,996127,999540,998213,992558,995388,993147,999562,995847\n", + "Ground Truth: 993158,990620,996389,998647,991809,995410,995511,999715,996609,998827,992215,990731,995271,999240,997893,994404,993473,997195,994334,993690,997859,992971,996476,990249,996127,993702,999540,990782,991447,998213\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 245\n", + "Returned IDs: 998913,995633,990126,990340,993219,996809,998495,990359,995289,995558,994382,993209,997975,994742,996226,994806,997209,996930,997023,996219,994835,995211,992566,996232,996328,992827,999646,993747,990621,995438\n", + "Ground Truth: 998913,995633,990126,990340,993219,996809,998495,990359,995289,995558,994382,997723,993209,997975,994742,998188,996226,994806,997209,996930,997023,996219,994835,996411,995211,992566,996232,996328,992827,995401\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 246\n", + "Returned IDs: 992646,999089,994670,996993,995336,999092,990595,998011,997479,996714,994197,991361,995793,992911,992183,998465,993596,997192,994757,998210,995824,999144,990275,994673,990647,995833,997605,994152,998951,995726\n", + "Ground Truth: 995374,990478,992646,999089,994670,996993,998385,995336,999092,990595,998011,995985,997479,996714,994197,991361,995793,992911,992183,998465,993596,997192,994757,998210,995824,999144,990275,994673,990647,995833\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 247\n", + "Returned IDs: 995473,992815,991045,992694,999616,998524,990588,995981,991900,997304,992874,997437,997320,993392,997913,998365,997353,999291,999725,998246,992074,998380,990535,998353,995701,991745,992384,994266,990862,995942\n", + "Ground Truth: 995473,992815,991045,992694,999616,998524,990588,997620,995981,991900,997304,992874,997437,993300,997320,993392,997913,998365,997353,999291,999725,998246,992074,998380,990535,998353,995701,991745,992384,992927\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 248\n", + "Returned IDs: 997377,995701,992851,994140,991349,995016,995813,996714,991154,991869,994922,995638,998831,994392,993738,991795,995245,999729,995184,990930,996368,995899,998200,991969,997928,995237,995555,999653,996001,994463\n", + "Ground Truth: 997377,995701,992851,997622,994140,991349,995016,995813,996714,991154,991869,994922,995638,998831,994392,993738,991795,995245,999729,995184,990930,996368,995899,998200,991969,997928,995237,990615,995555,999653\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 249\n", + "Returned IDs: 997529,995020,990711,998719,999500,992156,991189,994120,998495,994382,993725,996411,995633,990126,991881,991669,990435,991075,997485,998671,994730,998779,998982,993656,998862,990340,995002,993053,997914,998067\n", + "Ground Truth: 997529,995020,990711,998719,999500,992156,991189,994120,998495,994382,993412,993725,996411,997207,995633,990126,991881,991669,990435,991075,993439,997485,998671,994730,993747,998779,998982,993656,998862,990256\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 250\n", + "Returned IDs: 997723,995289,997046,993390,996867,998045,992160,995373,997466,993433,991603,996166,994409,993561,996322,997406,998070,995401,991415,996968,993831,997822,990836,999795,998165,996323,995702,996302,996360,991664\n", + "Ground Truth: 997723,995289,997046,993390,996867,998045,992160,993412,995373,997466,993433,991603,996166,994409,993561,996322,997406,998070,995401,996042,991415,996968,993831,997822,990836,999795,998165,996323,995702,996302\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 251\n", + "Returned IDs: 999826,997395,990790,999020,998080,994612,994170,996245,997184,993931,995807,997986,998820,994770,997301,995163,998049,992790,990243,997189,994280,993149,999967,991698,999898,995521,997918,993933,994774,997168\n", + "Ground Truth: 999826,997395,990790,999020,998080,994612,994170,996245,997184,993931,995807,997986,992879,998820,994770,997301,995163,998049,992790,990243,997189,994280,998014,993149,992683,999967,991698,999898,995521,997918\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 252\n", + "Returned IDs: \n", + "Ground Truth: 992198,990084,997407,991432,995631,997548,996742,992494,998754,995510,992459,992824,992462,996196,992095,997758,998693,997055,997547,998061,997931,997775,996065,990192,997744,996663,991977,993828,995955,998614\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 253\n", + "Returned IDs: 994020,995168,995231,990998,999783,993504,991691,995701,995695,997965,998959,998855,993342,991459,995780,999724,998623,994105,994906,997506,993076,995894,997618,997977,994365,998831,992862,997906,994125,999653\n", + "Ground Truth: 994020,995168,995231,990998,999783,993504,991691,995701,995695,997965,998959,998855,993342,991459,995780,993821,999724,998623,994105,994906,997506,993076,995894,997618,998514,997977,994809,994365,998831,992862\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 254\n", + "Returned IDs: 996633,994830,991885,992227,990735,992006,991390,993335,996603,996328,997026,998055,996314,995735,994806,996495,993725,994682,999496,993560,993100,998761,991382,994751,994960,990484,995002,995191,991610,998610\n", + "Ground Truth: 996633,994830,991885,992227,990735,992006,991390,993335,996603,996328,997026,998055,996314,998716,995735,994956,994806,996495,993725,994682,999496,993560,993100,998761,991382,994751,994960,990484,995002,995191\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 255\n", + "Returned IDs: 991817,998061,999763,994163,997614,996380,991432,999908,991871,993706,992824,997775,998308,992423,994270,991064,990538,990908,990894,995503,995459,990192,995983,999639,992462,995737,991550,991923,993839,990096\n", + "Ground Truth: 991817,998061,999763,994163,997614,996380,991432,999908,991871,993706,992824,997775,998308,992423,994270,991064,990538,990908,990894,995503,995459,992280,990192,995983,999639,992462,995737,991550,991923,993839\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 256\n", + "Returned IDs: 994125,995516,990897,990005,998697,995536,996918,993560,998758,997305,998536,994879,991530,997586,995672,991052,991630,993055,993948,998623,995963,991038,997752,997420,995701,993809,995455,997506,995237,995783\n", + "Ground Truth: 994125,995516,990897,990005,998697,995536,996918,993560,995401,997429,998758,997305,998536,994879,991530,997586,995672,991052,991630,993055,993948,998623,992293,995963,991038,997752,997420,995701,993809,995455\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 257\n", + "Returned IDs: 991720,993143,998921,995928,997370,991743,999077,992384,993785,992147,995336,995949,997918,996592,999710,997002,993260,995826,990366,997301,995515,996053,991249,994964,997661,993080,998090,995886,996504,994334\n", + "Ground Truth: 991720,993143,998921,995928,997370,991743,999077,992384,993785,998294,992147,995336,995949,997918,996592,999710,997002,993260,999109,995826,990366,997301,996829,995515,991626,996053,991249,994964,997661,993080\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 258\n", + "Returned IDs: 990827,990043,998983,998937,997370,996840,998551,997789,995988,999620,998734,991196,994922,999898,990858,995515,993282,998236,991348,997255,993647,992169,993143,993237,995443,996100,992641,996681,990332,998738\n", + "Ground Truth: 990827,990043,998983,998937,990849,997370,996840,998551,997789,995988,999620,994735,998734,991196,994922,999898,990858,997445,995515,993282,998236,991906,991348,997255,990500,993647,992169,993143,993237,995443\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 259\n", + "Returned IDs: 996219,993168,995876,993637,995001,998343,990173,998495,997331,999673,996067,990784,999430,997992,991037,995562,996549,996062,994279,992589,995438,990323,999748,990891,992576,990802,996408,997555,990958,993906\n", + "Ground Truth: 996219,993168,995876,995001,993637,999647,998343,993499,990173,999009,995689,998495,997331,999673,996067,990784,999430,997992,991037,995562,996549,991758,996283,996062,994279,991109,992589,995438,990323,991372\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 260\n", + "Returned IDs: \n", + "Ground Truth: 990796,995344,999829,992056,998855,990167,996249,999555,991038,999375,991037,998383,997332,997378,990891,992664,994193,999435,998335,991758,996702,994105,999430,996809,995895,995562,990194,998495,998594,996001\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 261\n", + "Returned IDs: 998917,997220,994617,994491,997826,995259,998782,998139,991727,995588,998983,994534,991999,992085,998928,995334,999535,991806,993340,990809,994528,994785,999147,995316,993143,999592,993282,995788,993570,994318\n", + "Ground Truth: 998917,997220,994617,994491,997826,997136,995259,998782,998139,991727,995588,998983,994534,992255,994741,990822,991999,999469,992085,998928,995334,999535,991806,991405,993340,990809,994528,998236,990890,994785\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 262\n", + "Returned IDs: 995345,993591,990368,990889,990928,995283,997933,994148,992415,990089,994379,999126,990918,997822,994663,997319,993053,997386,998165,996147,996906,998162,994783,992894,997529,994021,998765,995289,995295,994689\n", + "Ground Truth: 995345,993591,990368,990889,990928,995283,997933,994148,992415,990089,994379,999126,990918,997822,994663,997319,993053,997386,993886,998165,991923,991973,996147,996906,998162,994783,991537,992894,996382,997529\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 264\n", + "Returned IDs: 991137,996653,999533,998302,993845,997320,997921,999109,993579,996726,997381,995841,994978,999338,994332,999827,991714,997984,991787,992318,993807,992228,995659,992647,998797,990146,992896,992617,995372,993484\n", + "Ground Truth: 991137,996653,999533,998302,993845,997855,997320,997921,999109,993579,996726,990179,993066,997381,995569,995841,994978,999338,994332,999827,991714,997984,991787,992318,993807,992228,995659,998445,992647,998797\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 266\n", + "Returned IDs: 999356,998353,995981,998163,993853,998246,997728,992296,994879,990231,998662,992074,996075,993529,993186,990708,996609,994100,996384,992848,999740,997192,998950,998332,991721,992661,998119,997305,998697,992546\n", + "Ground Truth: 999356,998353,995981,998163,993853,998246,997728,992296,990453,994879,995755,990231,998662,991782,992074,996075,993529,993186,991913,990708,996609,994100,999761,996384,992848,990799,991895,999740,997906,997192\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 267\n", + "Returned IDs: \n", + "Ground Truth: 991685,991021,995945,997453,996062,997541,993472,998416,990899,990407,992182,997813,993665,992780,991633,990382,998593,998158,995895,995834,996566,996071,993931,990104,992664,994925,995421,999435,991857,999827\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 268\n", + "Returned IDs: 992944,995360,991848,993781,990043,999784,998991,994543,994341,991013,999210,997135,993708,998950,993260,996193,996802,998180,999122,997457,993279,995203,997379,992724,998123,993215,992907,997471,994171,992950\n", + "Ground Truth: 992944,995360,991848,993781,996829,990043,999784,998991,994543,994341,991013,999210,999577,997135,993708,998950,993260,996193,991416,996802,998180,990182,992168,995375,999122,996007,993365,997037,993633,997457\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 269\n", + "Returned IDs: 996429,997918,995701,998932,994276,996217,998303,991530,994221,993260,990409,993055,995932,990930,999384,994754,992168,998697,997897,990751,996196,993504,995203,992571,997586,990494,991332,999860,993087,992163\n", + "Ground Truth: 996429,997918,995701,998932,994276,995785,996217,992047,998303,991530,994832,994221,993260,990409,993055,995932,990930,998123,999384,994754,992168,998697,997897,990751,996196,993504,995203,994514,990425,992571\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 270\n", + "Returned IDs: 992562,997925,999639,995701,995005,999041,995886,991154,993809,996067,998200,991459,997337,996918,999219,991609,992927,994510,997918,995932,994081,993673,990897,995980,999444,999784,999122,998536,999384,997586\n", + "Ground Truth: 990530,992562,997925,999639,995701,995401,995005,999041,990588,993821,995886,990217,991154,993809,990930,996067,998200,991459,993834,997337,999680,996918,992933,999219,991609,992927,994510,997918,995636,995932\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 271\n", + "Returned IDs: \n", + "Ground Truth: 990736,995985,993387,995942,999263,997301,993250,996785,990595,991444,994978,997913,995841,998692,999291,997543,997972,993562,998170,990390,992709,997539,997704,991008,996218,992107,997192,997620,997320,998960\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 272\n", + "Returned IDs: 995208,996968,996867,998297,995401,994794,999530,992169,990432,995274,994409,995373,997191,990561,997871,998045,994301,995515,993143,996727,998080,991052,997723,996504,996769,995783,992364,992944,996903,990995\n", + "Ground Truth: 995208,996968,996867,998297,995401,990337,994794,999530,992169,993412,990432,995274,994409,998536,995373,990561,997191,994677,997871,998045,994301,995515,997775,997403,995374,993143,996727,998080,991052,996438\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 273\n", + "Returned IDs: 990072,999384,998861,997666,990096,997906,996623,997685,991459,995780,991471,993275,997265,999362,998623,996209,998536,995701,993561,999272,996733,994105,999827,999319,994955,997576,999736,995536,999194,990768\n", + "Ground Truth: 990072,999384,998861,997666,990096,997906,996623,991188,997685,991459,995780,998419,991471,993275,997265,999362,998623,993821,996209,998536,995701,990194,992977,998383,999653,993561,999272,996733,994105,994906\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 274\n", + "Returned IDs: 995702,995907,994380,996867,991161,993044,993188,997675,993597,996042,992576,994419,992212,993313,991794,999528,998106,991011,996388,997289,997822,992894,998165,995521,995666,999619,997020,991722,992413,993127\n", + "Ground Truth: 995702,995907,994380,996867,991161,993044,993188,997675,993597,996042,992576,994419,992212,993313,991794,999528,998106,991011,996388,997289,997822,992894,998165,995521,995666,999619,990725,997020,991722,992413\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 275\n", + "Returned IDs: 990020,997320,990532,997661,997918,993785,991795,992950,998302,996646,999446,995852,996075,993906,999444,994171,997227,991984,999990,994081,993561,999473,996001,996028,992546,995203,993157,993484,995993,994964\n", + "Ground Truth: 990020,997320,991989,990532,997661,997918,990207,993785,999219,991795,992950,991829,998634,998302,996646,999446,995852,996075,991311,993906,993412,999444,994171,997246,997227,997906,996735,994307,991984,999990\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 277\n", + "Returned IDs: 992546,990515,991431,999180,992339,992994,997614,990958,998246,990118,997305,994757,997437,996075,998797,991479,995981,994079,997320,990660,998562,994776,997543,997470,991045,997913,993662,998101,992119,997192\n", + "Ground Truth: 992546,990515,991431,993392,992583,999680,992238,999180,992339,992994,997614,990958,994653,998246,990118,997305,994757,997437,996075,998797,991479,995981,994079,997320,990660,998562,997976,994776,997543,997470\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 278\n", + "Returned IDs: 994427,990231,994403,992969,995697,998550,994171,996646,999710,994450,992119,993662,997320,994699,990135,999167,997354,990910,996540,990066,990182,997896,990957,999328,994961,991260,994604,997913,993264,990650\n", + "Ground Truth: 994427,990231,994403,996921,992969,993220,995697,992363,991743,998550,994171,996646,999710,994450,992119,993662,993278,997320,994699,991320,994048,995351,990135,995942,999167,994192,997354,990910,996540,990066\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 279\n", + "Returned IDs: 996388,993390,999080,995665,998979,996379,993561,991415,995289,993144,997406,995701,996719,991857,998128,992125,994811,991954,998045,998874,991630,997787,991113,996924,991011,997046,994113,996181,998165,991603\n", + "Ground Truth: 996388,993390,999080,995665,998979,996379,993561,991415,995289,993144,997406,995701,996719,991857,998128,992125,994811,995761,991954,996294,998045,998874,991630,995864,997787,991113,996924,991011,997046,994113\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 280\n", + "Returned IDs: 992628,996016,996794,995535,991382,993391,996474,999280,998803,996184,998084,994525,996938,995880,999205,997798,995735,992237,997084,996839,999634,992867,992073,998153,992754,995346,996650,994472,992100,995191\n", + "Ground Truth: 992628,996016,996794,995535,991382,993391,996474,999280,998803,996184,998084,994355,994525,996938,995880,999205,997798,995735,992237,997084,996839,999634,992867,992073,998153,992754,995346,996650,994472,992100\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 281\n", + "Returned IDs: 991269,993209,997023,993586,999957,996607,995778,997723,991507,997544,994496,999054,998188,991639,999415,999047,991140,998779,991136,999563,996867,997975,990372,992413,994881,992582,995289,997832,995401,995665\n", + "Ground Truth: 991269,993209,997023,993586,999957,996607,995778,997723,991507,997544,994496,999054,998188,991639,991943,999415,999047,991140,998779,997101,991136,999563,996867,992334,997975,990372,998295,992413,994881,992582\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 282\n", + "Returned IDs: 991820,993080,998720,991981,993342,995906,993809,990860,996911,999444,998593,995852,990532,999916,993290,997969,998710,995002,999922,996001,997790,993906,994636,990622,990078,992632,994783,993637,995184,995256\n", + "Ground Truth: 991820,993080,998720,991981,993342,995906,993809,998514,990860,996911,999444,995160,998593,995852,990532,996490,999916,990866,993290,999272,997969,998710,995002,999922,996001,991795,999647,997790,995961,995636\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 283\n", + "Returned IDs: 999912,998039,997920,996996,995384,995906,990737,997028,991565,997410,997529,990974,997345,995526,990315,999949,998169,993637,996219,994964,994293,999261,994783,999647,993378,991831,993168,998382,999151,990728\n", + "Ground Truth: 999912,998039,997920,996996,995384,995906,990737,997028,991565,997410,997529,990974,997345,995526,990315,999949,998169,993637,996219,994964,995222,994293,991359,999261,990891,994783,999647,993378,991831,995401\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 284\n", + "Returned IDs: \n", + "Ground Truth: 996524,993012,990366,997811,991373,997466,998568,993696,999202,992330,990866,996857,993628,992238,993834,997586,993823,997690,997806,995132,992019,998403,992068,995268,995516,999598,994879,990991,992521,997287\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 285\n", + "Returned IDs: 993805,990256,991052,995516,990897,994869,996406,999646,996811,994956,991982,996918,996695,995432,996762,997126,998318,990563,992795,993948,994125,991038,996861,991860,998123,992558,994510,993143,999820,994558\n", + "Ground Truth: 993805,990256,991052,993255,994714,995516,990897,994869,996406,999646,999676,993673,996811,994956,991982,996918,996695,992979,992147,995432,996762,997126,998318,991857,991824,990563,992795,993948,991977,994125\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 286\n", + "Returned IDs: 998779,998862,997640,993412,993293,997204,996727,999026,994382,996436,997319,999021,994409,995401,992169,995216,997355,994436,995374,995940,995274,997871,997832,995685,990675,990940,995200,996379,996368,997320\n", + "Ground Truth: 998779,998862,997640,993412,998999,993293,997204,996727,993831,999026,994382,996436,997319,997732,999021,991923,994409,995401,992169,995216,997355,994436,993280,995374,995940,991743,993226,996042,995274,997871\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 287\n", + "Returned IDs: 994313,990696,995783,999496,999871,990872,996087,997914,991319,990386,998293,999105,997919,997752,996362,990083,993678,999179,994051,993621,990061,995464,993727,993441,996814,993080,992571,996429,996825,991269\n", + "Ground Truth: 994313,990696,995783,999496,999871,990872,996087,997914,991319,990386,998293,999105,997919,997752,997506,996362,998388,990083,997429,993678,999179,994051,993621,992617,990061,995464,993727,993441,993080,996814\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 288\n", + "Returned IDs: 996249,991977,990796,994193,994605,995760,995727,998855,996882,999281,999375,998450,993316,990487,993681,992435,995357,993500,990776,992376,990206,992056,995618,992558,993715,990998,996809,995965,992052,997309\n", + "Ground Truth: 996249,991977,990796,994193,994605,995760,995727,998855,996882,999281,999375,998450,993316,990487,993681,992435,995357,993500,990776,992376,990206,992056,995618,992558,993715,990998,997104,996809,995965,992052\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 289\n", + "Returned IDs: 995289,993844,998014,992488,992289,994204,993433,995766,993143,998536,997319,993762,996867,994783,991840,997897,993232,995633,992052,998913,990941,999453,997822,996809,995788,996769,990882,990681,990340,998165\n", + "Ground Truth: 995289,993844,997403,998014,992488,992289,994204,993433,995401,995766,996438,993143,993673,998536,997319,993762,996867,994783,996930,991840,993232,997897,995633,992052,998913,990337,990941,992407,993209,999453\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 290\n", + "Returned IDs: 996657,993848,998593,990662,990811,994587,992293,992367,991185,998549,995289,991603,993390,990255,998779,993738,998165,996001,992703,991835,997822,994809,996759,996368,996147,998862,995158,991630,993561,996277\n", + "Ground Truth: 996657,993848,998593,990662,990811,994587,996864,992293,995401,992367,991185,998549,995289,991603,997723,993390,990255,998779,993738,998165,996001,992703,991835,997822,999587,990930,994809,996759,996368,996294\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 291\n", + "Returned IDs: 991795,996987,992448,996187,994409,998143,998361,998205,995864,994954,994872,996256,993003,999250,999926,998941,991969,998756,991349,997868,990341,993672,990603,995016,996162,998527,996919,991892,998278,994865\n", + "Ground Truth: 991795,996987,992448,996187,994409,998143,998361,998205,995864,994954,994872,996256,993003,999250,999926,998941,991969,998756,991349,997868,990341,999866,993672,990603,995016,996162,998527,996919,991892,998278\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 292\n", + "Returned IDs: 993127,993400,997674,995891,995470,999884,995423,998651,992448,995032,994273,991276,998147,999918,996621,999255,992967,996472,995122,994413,998321,999043,994113,995905,992048,994419,997675,994409,991699,992753\n", + "Ground Truth: 993127,993400,997674,995891,995470,999884,995423,998651,992448,995032,994273,991276,998147,999918,996621,999255,992967,996472,995122,994413,998321,999043,994113,995905,992048,994419,997675,994409,991699,998973\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 293\n", + "Returned IDs: 994672,994247,993715,999139,994577,995801,995349,993504,991552,993055,995839,993139,991245,995737,996884,993181,991648,994351,993609,999079,990885,995622,992747,992052,991332,999806,993489,999927,991605,999526\n", + "Ground Truth: 994672,994247,993715,999139,994577,995801,995349,993504,991552,993055,995839,993139,991245,995737,996884,993181,994743,991648,994351,993609,999079,990885,995622,997076,992747,992052,991332,999806,993489,999927\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 294\n", + "Returned IDs: 995139,995905,995507,999619,996038,991027,995470,996323,993400,997046,999578,997832,995547,992448,997139,993421,995923,997674,999508,990975,996867,994554,999530,993597,997252,996322,994278,991276,994409,994419\n", + "Ground Truth: 995139,995905,995507,999619,993468,992660,996038,991027,995470,996323,993400,997046,999578,997832,995547,997139,992448,993421,995923,997674,999508,990975,996867,994554,999530,990634,994002,993597,997252,996322\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 295\n", + "Returned IDs: 998294,998380,995949,999263,997301,994964,990561,998697,998090,997320,991530,995928,998029,997913,993342,999783,999444,995802,995672,998302,998365,997371,998692,990210,990366,998303,990146,999639,991743,990958\n", + "Ground Truth: 998294,998380,995949,999263,997301,994964,990561,998697,998090,997320,991530,991157,995928,998029,997913,993342,999783,999444,995802,995672,998302,997032,995638,998365,999331,997371,990310,998692,998607,997135\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 296\n", + "Returned IDs: 991720,992147,997842,998921,995372,992169,996915,991743,997192,995515,991889,998877,993080,991485,999698,997918,995927,992944,995443,994280,995771,995336,990432,998229,991503,994919,997984,995788,992942,993215\n", + "Ground Truth: 991720,999343,992147,997842,998921,995372,992169,996915,998259,993857,991743,997192,995515,991889,998877,993080,991485,999698,997918,995927,992944,991406,995443,994088,994280,995771,990648,995336,990432,992309\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 297\n", + "Returned IDs: 999616,995981,997437,999180,990588,999596,992296,993168,997320,995336,997759,997353,998353,995372,992748,994695,998828,991045,997875,990339,996075,998607,995184,993080,997309,994894,994964,990957,995927,997018\n", + "Ground Truth: 999616,995981,991720,995792,997437,999099,999180,990588,999596,992296,993168,998567,997320,994916,990778,995336,997759,997353,998353,995372,999527,992748,994695,993893,998828,991045,997875,990339,996075,998607\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 298\n", + "Returned IDs: 994745,991980,998716,998803,998903,991911,994830,993335,990730,996153,993101,998584,998030,991554,993568,996603,990678,991187,995735,997952,992495,996357,994683,992598,996183,991081,991297,993522,998440,998949\n", + "Ground Truth: 994745,991980,998716,998803,998903,991911,994830,993335,990730,996153,993101,998584,998030,991554,993568,996603,990678,991187,995735,997952,999786,992495,996357,994683,992598,996183,991081,998484,996503,991297\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 299\n", + "Returned IDs: 998039,991869,990076,993561,995923,996028,990845,995289,998633,990096,999729,993164,994925,990930,994853,994409,992851,994140,995813,997716,995526,991958,996001,990634,999516,995401,996062,991795,997386,997170\n", + "Ground Truth: 998039,991869,990076,993561,995923,996028,990845,995289,998633,990096,999729,993164,994925,990930,994853,994409,992851,994140,995813,997716,995526,991958,996001,990634,999516,995401,995678,996062,991795,997386\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 300\n", + "Returned IDs: 990602,996367,993583,993534,999305,990226,996016,992306,991432,993255,998357,999540,991605,992113,997547,993839,998652,993427,999479,997127,995618,995631,991977,993380,996742,999908,990423,994254,994560,994472\n", + "Ground Truth: 990602,996367,993583,993534,999305,990226,996016,992306,991432,993255,998357,999540,991605,992113,997547,993839,998652,993427,999479,997127,995618,995631,991977,993380,995542,996742,999908,990423,994254,994560\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 301\n", + "Returned IDs: 997585,994922,991392,995557,999620,996402,995057,990043,990090,994964,993484,996563,998837,993389,999437,993237,999484,994789,991150,992944,993355,993168,990229,990585,998765,992907,990690,994685,996219,990483\n", + "Ground Truth: 997585,994922,991392,995557,999620,996402,995057,990043,990090,994964,993484,996563,998837,993389,999437,991582,993237,999484,994789,991150,992944,993355,996419,993168,992346,993775,990229,990585,998765,997876\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 302\n", + "Returned IDs: 996606,993534,993427,995152,991983,995618,992113,994679,992857,993380,992558,990279,995710,999540,997849,992496,990345,997547,997917,991046,996750,998151,992092,998840,991026,990125,996367,993583,992306,999874\n", + "Ground Truth: 996606,993534,993427,995152,991983,995618,992113,994679,992857,993380,992558,990279,995710,999540,997849,992496,991058,990345,997547,997917,991046,996750,992335,998151,991871,992092,998840,991026,990125,996367\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 303\n", + "Returned IDs: 995459,998142,992824,997775,999171,994679,991432,991817,993706,993077,991550,994276,993583,997864,999908,992392,995503,993647,993828,995983,996219,992280,999639,995737,998061,990765,993260,990538,993891,999560\n", + "Ground Truth: 995459,998142,992824,997775,999171,994679,991432,991817,991370,993706,993077,991550,994276,993583,997864,999908,992392,995503,993647,993828,997906,995983,996219,992280,999639,995737,998061,990765,991923,990538\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 304\n", + "Returned IDs: 999504,997388,997157,996727,990797,991923,991082,998094,997640,999960,994406,999374,997083,995401,998529,997634,998559,999820,990475,998607,993412,990275,991977,994514,998779,997748,998179,999462,995369,991334\n", + "Ground Truth: 999504,997388,997157,996727,990797,991923,991082,998094,997640,999960,994406,999374,997083,995401,998529,997799,997634,998559,999820,990475,998607,995320,993412,990275,991977,994514,998779,997748,998179,999462\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 305\n", + "Returned IDs: 990644,997702,997002,991392,991485,990999,992979,999219,991459,993355,999453,995894,995875,993848,991157,998385,999977,996425,994946,998247,998401,995154,995515,994334,995016,998303,994906,996840,996368,992648\n", + "Ground Truth: 990644,997702,997002,991392,991485,990999,997533,994362,995374,992979,991894,999219,991459,993355,999453,995894,995875,993848,991157,998385,992750,999977,996425,994946,991151,998247,998401,995154,992047,995515\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 306\n", + "Returned IDs: 997475,994891,993611,994282,993821,991153,998065,996535,991066,990301,993046,990479,990961,996728,999775,999832,991597,994316,991111,994280,999205,996495,996431,990774,997919,995600,998221,991709,994196,998537\n", + "Ground Truth: 997475,994891,993611,994282,993821,991153,998065,996535,991066,990301,993046,990479,990961,996728,999775,999832,991597,994316,991111,998279,994280,998320,999205,996743,996495,996431,990774,997919,995600,998221\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 307\n", + "Returned IDs: 995260,995755,992546,994266,990925,995132,991829,997737,996001,997320,995927,993168,994776,997796,999444,991743,990146,996075,990958,992994,992147,996714,990096,993648,998326,996219,998365,993484,999647,998246\n", + "Ground Truth: 995260,995755,992546,994266,990925,995132,991829,997737,996805,996001,997320,995927,993168,994776,997796,999444,991743,990146,991582,996075,990958,992994,992147,997876,996714,990096,993648,998326,996219,998365\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 308\n", + "Returned IDs: 993426,990191,992451,991873,999271,999904,995623,995045,997292,997301,997984,996516,998921,993484,990546,995306,992709,990512,998760,991720,997397,996771,998425,999110,992107,992027,991916,999533,994618,997320\n", + "Ground Truth: 993426,990191,992451,991873,999271,999904,995623,995045,997292,997301,997984,996516,998921,998603,993484,990546,995306,995865,993048,992709,990512,998760,991720,997397,996771,998425,999110,992107,992027,991749\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 309\n", + "Returned IDs: 997233,997059,994205,998725,994013,996308,999130,990241,990662,996028,998982,999099,997452,999570,997554,996657,994382,991382,995295,995216,996001,998752,996147,999623,998055,996411,995544,995456,999342,995158\n", + "Ground Truth: 997233,997059,994205,998725,993857,994013,996308,999130,990241,990662,996028,995600,998982,999099,997452,999570,994131,997554,996657,993909,994382,991382,995295,995216,996001,998752,999623,996147,998055,998156\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 310\n", + "Returned IDs: 993567,998104,990872,994711,998293,994909,996435,993727,992457,992569,993799,992052,999796,990116,997897,992437,998162,997752,990956,992981,993180,993497,996825,991245,992028,995766,995421,994353,994435,993347\n", + "Ground Truth: 993567,998104,990872,994711,998293,999190,994909,996435,993727,994300,992457,992569,997949,993799,992052,999796,990116,997897,992437,998162,997752,990956,992981,992759,993180,993497,996825,991245,997506,992028\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 311\n", + "Returned IDs: 992041,992124,999639,997258,992238,990453,993696,998758,998260,992167,992225,991984,995516,993834,992927,994809,993283,990949,997370,997320,999041,998950,991157,992521,991459,990862,996500,995899,996075,992896\n", + "Ground Truth: 992041,992124,999639,997258,992238,990453,992497,993696,998758,998260,992167,992225,991984,995516,993834,992927,994809,993283,990949,997370,997320,999041,998950,991157,992521,991459,990862,996500,995899,996075\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 312\n", + "Returned IDs: 990821,990310,991840,995895,991157,993717,996809,991015,995633,993391,994922,993143,996408,998950,996840,994830,991848,993342,999453,999784,996328,994742,999122,992169,992612,997822,994964,993260,997918,990080\n", + "Ground Truth: 990821,990310,991840,995895,991157,990271,993717,996809,991015,995633,990941,993391,997331,994922,993642,993143,996408,998033,998950,996840,994830,991848,993342,999453,999784,996328,994742,999122,992169,992612\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 313\n", + "Returned IDs: 996936,999337,998794,991805,999157,993890,996944,998926,992499,998109,997047,991915,997270,995647,996006,996650,996796,992780,993472,995690,994558,992872,993721,994452,996475,994983,994832,991179,995596,996219\n", + "Ground Truth: 996936,999337,998794,991805,999157,993890,990780,996944,998926,992499,998109,997047,991915,997270,995647,996006,996650,997561,996796,992780,993472,995690,992986,994558,992872,993721,994452,996475,997257,993772\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 314\n", + "Returned IDs: \n", + "Ground Truth: 998728,998232,992733,994440,996075,990142,995596,992751,991984,993902,997291,997661,998498,995203,992671,991461,994626,999426,996317,994341,990123,996735,998654,999132,990083,996741,992119,990914,990697,994115\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 315\n", + "Returned IDs: 992771,997395,993566,990194,993106,996393,999994,993931,999673,995895,992685,996408,990951,999543,995620,999555,991037,999430,990407,993224,998536,998006,990796,990434,993011,996395,998080,991148,990167,991052\n", + "Ground Truth: 992771,997395,993566,990194,993106,996393,999994,993931,999673,995895,992685,991188,996408,990951,999543,995620,999555,991037,999430,990407,993224,998536,998006,996762,990796,990434,993011,995211,998383,995401\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 316\n", + "Returned IDs: 999932,995269,993465,996046,996903,995685,997046,994819,990459,997918,994804,999026,993407,993935,995444,995373,993433,997002,996734,999000,997822,996323,994334,990940,991451,995928,991415,995204,990215,998023\n", + "Ground Truth: 999932,995269,993465,996046,996903,995685,997046,994819,990459,997918,994804,999026,993407,990995,993935,995444,993412,995373,991698,993433,997002,992933,996734,999000,997822,996323,994334,990940,991451,999089\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 317\n", + "Returned IDs: 999139,991382,995895,998671,997506,995191,996328,994142,990392,999740,994280,994830,993053,998716,998495,991633,997933,996013,991672,992680,994472,993391,992636,992445,999086,999430,991390,995476,995295,996537\n", + "Ground Truth: 999139,991382,995895,998671,997506,995191,996328,994142,990133,990392,999740,994280,994830,999377,993053,998716,998495,991633,997933,996013,998030,991672,992680,994472,990293,993391,996775,992636,999441,992445\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 318\n", + "Returned IDs: 991144,993719,990257,993917,992636,993715,996986,990438,990951,997168,999334,996357,993150,993637,995991,998076,998716,998561,990089,997879,998348,996408,997790,996328,995002,994834,993139,996201,990289,999827\n", + "Ground Truth: 991144,993719,990257,993917,992636,993715,996986,990438,990951,997168,999334,996357,993150,993637,995991,998076,998716,998561,990089,997879,998348,996408,997790,996328,995002,994522,994834,991081,997669,993139\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 319\n", + "Returned IDs: 992747,998303,993489,991037,992571,990350,998401,999840,990409,991332,995536,990930,996217,997002,997977,992082,995894,996062,993504,994200,995203,993260,990885,990096,991530,997368,995025,995823,991425,992823\n", + "Ground Truth: 992747,998303,993489,991037,992571,990350,998401,999840,990409,991332,995536,990930,996217,994468,997002,997977,992082,995894,996062,993504,994200,995203,993260,990885,990096,991530,996677,997368,995025,995823\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 320\n", + "Returned IDs: 993809,998080,998495,996371,992571,993255,996370,992318,990083,998950,993696,992847,993515,995932,990532,999701,997586,992944,996805,997184,991412,990711,998416,990776,999179,992748,995552,996918,996809,995289\n", + "Ground Truth: 993809,998080,998495,995401,996371,992571,993255,991052,996370,992318,990083,998950,993696,992847,993515,997723,992753,995932,990532,993478,999701,998221,997586,992944,996805,997184,991412,993444,990711,998416\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 321\n", + "Returned IDs: 990999,998990,998179,995401,997258,991459,997191,996379,996198,996727,993561,990354,993848,999219,992684,991392,991630,999021,994946,994319,996247,999639,990838,995994,990561,992979,997640,993355,997158,997380\n", + "Ground Truth: 990999,998990,998179,995401,997258,991459,997191,996379,996198,996727,993561,990453,997799,990354,993848,999219,992684,991392,991630,999021,994946,993412,994319,996247,999639,990838,991792,999228,995994,990561\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 322\n", + "Returned IDs: 990866,997379,997586,993708,996397,993637,993552,999308,999647,999500,995701,997013,992068,991157,997811,992944,991808,997555,999598,998180,992682,995455,999473,999337,998550,991015,994341,991348,995360,994307\n", + "Ground Truth: 990866,997379,997586,993708,996397,993637,996429,993552,999308,999574,999647,999500,995701,997013,992068,990547,991157,997811,992944,991808,999216,992522,992047,997555,999598,997496,998180,998790,992682,995455\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 323\n", + "Returned IDs: 993160,995062,998080,999961,992150,997634,998806,997061,997395,998316,998650,992677,990131,993255,993263,994576,990789,995886,996273,992161,992604,994774,996139,993122,992493,991360,992558,992148,993740,990957\n", + "Ground Truth: 993160,995062,998080,993971,999961,992150,997634,998806,997061,997395,998316,998650,992677,997429,990131,993255,993263,994576,990789,995886,996273,990103,992161,992604,994774,996139,993122,992493,998528,991360\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 324\n", + "Returned IDs: 991825,996028,994689,995823,998140,998825,993378,997051,990811,997373,994012,991840,998982,993906,992201,997417,993890,998382,994307,995089,993725,991989,992780,999446,999740,995680,999286,996556,999613,991711\n", + "Ground Truth: 991825,996028,994689,995823,998140,998825,993378,997051,995649,990811,991426,997373,994012,991840,998982,993906,992201,997417,993890,998382,994307,995502,997541,995089,993725,991989,992780,999446,999740,995680\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 325\n", + "Returned IDs: 994630,996961,997525,994699,993518,991895,997913,998246,999710,992546,992709,994939,994932,995949,994757,998744,996785,997599,998332,990397,997535,990925,990509,991873,992119,991479,996890,991379,994427,992107\n", + "Ground Truth: 994630,996961,997525,999761,994699,993518,991895,996732,997913,998246,999710,992546,992709,994939,998016,994932,995949,994757,998744,997599,996785,998332,990397,992502,997535,990925,990509,991873,992119,991479\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 326\n", + "Returned IDs: 997976,998170,994894,997479,999182,994757,997320,999144,997918,995981,995279,995886,990231,994100,991379,990660,997571,995336,990958,994334,997192,990647,990588,995656,997605,997437,994280,993492,998951,995274\n", + "Ground Truth: 997976,998170,994894,997479,999182,994757,991782,997320,999144,997918,993673,995981,995279,995886,990231,994100,991379,990660,997571,998011,995336,990958,998163,994819,994334,997192,999527,991181,990647,990588\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 327\n", + "Returned IDs: 995274,996657,996001,995216,998302,999026,998779,996810,998952,990974,990845,996759,998862,995289,998058,990957,992792,999099,997529,996219,999343,993168,994730,996727,997059,999453,997554,993831,996147,995873\n", + "Ground Truth: 995274,996411,990293,996657,993412,996001,993909,995216,993003,998302,999026,998779,996810,998952,990974,991995,990845,996759,998862,995289,992120,995899,998058,999835,999587,990957,992792,997969,999099,997529\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 328\n", + "Returned IDs: 998257,995373,991204,997110,990657,993115,997568,994643,998858,994454,997589,999129,993795,991987,999530,991386,995401,993000,996393,997675,997683,992762,999136,998536,991052,990096,992861,998697,997918,990005\n", + "Ground Truth: 998257,995373,991204,997110,990657,993115,997568,994643,998858,994454,997030,997589,999129,993795,991987,999530,991386,995401,993000,996393,997675,997683,992762,999136,998536,991052,990096,992861,998697,995453\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 329\n", + "Returned IDs: 996001,992052,996657,996277,991977,997453,996505,991349,993280,999491,994743,998536,997168,993180,994409,990956,998593,993289,990930,994040,995200,992521,993561,994280,993931,992790,992718,995289,996042,990364\n", + "Ground Truth: 996001,992052,996657,996277,991977,997453,996505,991349,993280,999491,994743,998536,997168,993180,994409,995408,990956,998593,993289,990930,994040,995401,995200,992521,993561,994280,993931,999099,992790,992718\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 330\n", + "Returned IDs: 990255,997897,993232,990167,996657,998593,990361,991532,996395,998695,996809,993344,997752,990941,998786,992809,995167,993483,996001,996284,991255,990675,994587,996797,994451,996043,992437,999463,997131,990314\n", + "Ground Truth: 990255,997897,993232,990167,996657,998593,990361,991532,996395,996104,998695,996809,993344,997752,990941,998786,992809,995167,993483,996001,996284,991255,990675,994587,996797,994451,990211,996043,992437,999463\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 331\n", + "Returned IDs: 990154,998006,995815,999555,994181,997666,992771,995835,999673,990952,999543,992282,990951,994834,999388,994467,998861,996505,995655,994103,998067,995536,994756,997242,997427,994380,990537,997331,993931,995168\n", + "Ground Truth: 990154,998006,995815,999555,996728,994181,997666,992771,995835,999673,990952,990006,999543,997884,992282,990951,994834,999388,994467,998861,999350,999663,996505,995655,991596,994103,990429,998067,997358,995536\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 332\n", + "Returned IDs: 997452,991464,990614,998913,995295,993725,994742,991330,995633,997933,996116,994783,998779,994541,990701,995551,996411,995456,991303,998543,993053,990392,992434,997822,998495,992703,995289,998671,999302,997374\n", + "Ground Truth: 997452,991464,990614,998913,995295,993725,994742,991330,995837,995633,997933,996116,994783,998779,994541,990701,995551,996411,995456,991303,991537,999230,994580,998543,993053,990392,992434,997822,998495,992703\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 333\n", + "Returned IDs: 998109,999659,992780,997442,997296,992499,995476,991826,992070,991382,996809,995150,997270,996216,996650,993617,990039,998167,991633,998794,998495,991805,993342,999430,992872,998631,993391,998803,994927,994342\n", + "Ground Truth: 998109,999659,992780,997442,997296,992499,995476,991826,992070,991382,996809,995150,997270,996216,996650,993617,990039,998167,991633,998794,998495,991805,993342,999430,992872,999551,998631,993391,997318,998803\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 334\n", + "Returned IDs: 991377,992894,994419,995289,997386,997822,995702,995923,991923,993511,996080,990847,993561,996610,996867,991958,996360,997529,996745,996147,993433,994925,996322,995200,996127,994409,992130,997937,999782,996042\n", + "Ground Truth: 991377,992894,994419,995289,997386,997822,993412,995702,995923,991923,993511,996080,991603,990847,993561,996610,996867,991958,996360,997529,996745,996147,993433,994925,996322,995200,996127,994409,992130,997920\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 335\n", + "Returned IDs: 993079,993324,998289,999796,998543,997159,998104,994765,994119,992558,998180,990956,997420,990814,994742,993215,999179,990776,994541,995332,995438,995289,993293,990264,992832,997309,995401,998950,996210,994817\n", + "Ground Truth: 993079,993324,998289,999796,998543,997159,998104,994765,994119,992558,992933,998180,990956,993280,997420,990814,994742,993215,999179,998369,990776,994541,996370,995332,995438,995289,996178,993293,990264,992832\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 336\n", + "Returned IDs: 991454,998355,991177,993929,990103,990226,997732,994956,997244,998691,990712,993014,998631,991107,999304,994444,999673,996803,998343,994410,990678,990705,998495,997453,995382,999138,990951,991170,995919,992713\n", + "Ground Truth: 991454,998355,991177,993929,990103,990226,997732,994956,997244,993336,998691,996062,996362,990712,993014,998631,991107,999304,990408,994444,999673,997693,997807,998466,996803,991219,996942,998343,994410,994927\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 337\n", + "Returned IDs: 996531,998397,993747,990956,996276,999514,991747,998293,996867,992703,994119,994541,998165,996232,998486,992160,990116,997822,994444,999264,999646,990103,993727,997420,995159,999244,991145,995184,995144,994806\n", + "Ground Truth: 996531,998397,993747,990956,996276,999514,991747,998293,996867,998478,992703,994119,990466,994541,998165,996232,998486,992160,990116,997822,994444,999264,999646,990103,993727,996294,997420,995159,992992,999244\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 338\n", + "Returned IDs: 992112,990494,998715,992351,990885,997368,998018,997897,995932,996632,993504,992747,992825,997839,993171,997790,990818,995560,993260,995476,990697,990009,992780,991491,998473,992635,996216,994081,994280,993341\n", + "Ground Truth: 992112,990494,998715,992351,990885,997368,998018,997897,995932,996632,993504,992747,992825,997839,993171,997790,990818,992665,992137,995560,993260,995476,990697,990009,992780,995130,991491,998473,996424,992635\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 339\n", + "Returned IDs: 993209,997837,995778,995289,995766,995633,993997,994742,991185,992558,991507,994031,994743,999957,995106,996302,998748,992448,998188,993623,991044,997975,999415,997630,993003,995648,997319,994409,993265,997662\n", + "Ground Truth: 993209,997837,995778,995289,995766,995633,990361,993997,994742,991185,992558,995401,991507,994031,994743,999957,995106,996302,998748,995198,992448,998188,993623,991044,997975,991664,999377,999415,997630,993003\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 340\n", + "Returned IDs: 994662,994755,996276,995144,994073,995438,990956,999244,992315,993747,992934,997420,999601,992052,999514,998104,993805,996850,991747,993727,993742,993121,994541,999264,998293,999411,993448,999029,994336,993219\n", + "Ground Truth: 994662,994755,996276,995144,994073,995438,990956,999244,992315,993747,992934,997420,999601,992052,999514,998104,993805,996850,991747,993727,993742,993121,994541,999264,998293,999411,993448,999029,994336,995107\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 341\n", + "Returned IDs: 997386,995289,995923,994419,996080,993561,996867,998593,991603,998999,992894,997822,998165,997837,994925,995497,990811,995702,991958,992680,998382,993725,997046,998039,996322,995052,995200,996745,997406,993433\n", + "Ground Truth: 997386,995289,995923,994419,996080,993561,996867,998593,991603,998999,992894,997822,998165,997837,994925,995497,990811,995702,993412,991958,992680,998382,993725,995401,997046,998039,996322,995052,997630,995200\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 342\n", + "Returned IDs: 993364,992126,993885,990772,997380,990557,995841,994162,991212,991498,995372,997320,999982,995502,998444,994673,992788,995515,999453,996489,997973,997002,996001,996959,993032,994081,997301,997796,994088,990487\n", + "Ground Truth: 993364,992126,991994,993885,991792,990772,997876,997380,990557,993820,995841,994162,991212,991498,995372,997320,999982,995502,998444,994673,992788,995515,999453,996489,997973,999819,997002,996001,990724,996959\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 343\n", + "Returned IDs: 996442,991713,998260,996397,997158,999806,999731,996196,993785,998827,992423,998568,992927,994577,994163,990731,993071,991650,993504,997550,999598,996404,992041,995552,997977,991157,995820,999111,997320,997013\n", + "Ground Truth: 996442,991713,998260,996397,997158,999806,999731,996196,993785,998827,992423,998568,992927,994577,994334,994163,990731,993071,991650,993504,997550,999598,996404,992041,996979,995552,997977,993377,991157,995820\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 344\n", + "Returned IDs: 994277,993721,994233,994416,991409,996685,995899,992944,996024,997320,998950,991348,991416,998921,996210,996075,992318,995372,998913,996178,991650,991997,997305,995108,996001,991154,993018,991142,999701,995515\n", + "Ground Truth: 994277,993721,994233,994416,991409,996685,995401,997403,995899,992944,996024,997320,998950,991348,991416,998921,996210,993121,996075,994081,997448,990364,995372,992318,998913,996178,991650,991997,991743,997305\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 345\n", + "Returned IDs: 999827,996001,997320,994964,991498,993080,999453,994081,995841,992182,999099,994478,992395,993168,995216,997860,993280,997918,995274,990146,993289,995102,993484,991212,990532,993388,997371,990557,992469,999019\n", + "Ground Truth: 999827,996001,997320,994964,991498,993080,993454,997920,999453,994081,995841,992182,999099,999026,994478,992395,993168,995216,997860,998750,993280,997918,995274,990146,993477,993289,990772,993648,995102,991007\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 346\n", + "Returned IDs: 992907,995588,997465,998582,993091,999676,996840,995515,993622,999076,991840,990300,993355,990310,994922,991999,996686,993781,999077,994612,990432,993688,999620,991172,993143,991462,999784,993212,997496,999317\n", + "Ground Truth: 992907,995588,997465,998582,993091,999676,996840,995515,993028,993622,999076,991840,990300,993355,990310,994922,992979,999073,996352,999604,991999,996686,993781,999077,994612,990490,990432,993688,991160,992218\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 347\n", + "Returned IDs: 997909,997299,998890,996256,999925,993003,994409,997918,993537,991357,991795,992763,996815,996609,995864,994106,990087,990625,990432,999926,996613,994865,998205,991892,991927,997612,998361,991197,991220,994809\n", + "Ground Truth: 997909,997299,998890,996256,999925,993003,994409,995401,997918,993537,991357,991743,991795,992763,996815,996609,995864,997555,994106,990087,990625,994062,990432,999926,996613,997218,994865,996232,998205,991892\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 348\n", + "Returned IDs: 991487,991515,990630,994101,996562,996817,996496,999079,998250,993175,996461,992104,990203,995349,990834,993391,990160,991245,997666,995832,991897,998934,998022,997506,993483,995421,992407,990412,993489,996245\n", + "Ground Truth: 991487,991515,990630,994101,996562,996817,996496,999079,998250,993175,996461,992104,990203,995349,990834,993391,990160,991245,997666,999211,994952,998428,990712,995832,993599,999463,991897,998934,998022,997506\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 349\n", + "Returned IDs: 993754,997781,992604,995527,995807,999125,994774,999445,995105,999662,990915,998059,991051,999478,996691,992387,990131,998297,997212,990729,997462,997693,997110,993632,998528,992855,994539,995953,996206,996236\n", + "Ground Truth: 993754,997781,992604,995527,995807,999125,994774,999445,995105,999662,990915,998059,991051,999478,996691,992387,990131,998297,997212,990729,997462,997693,997110,993632,998528,992855,995625,994539,995953,996206\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 350\n", + "Returned IDs: 994142,997693,997309,990712,995214,991525,999179,990776,990956,997420,993453,996328,991257,994282,999091,997942,994193,993931,990705,993621,994119,995159,994092,992685,991723,994035,994835,996055,996633,993929\n", + "Ground Truth: 994142,997693,997309,990712,995214,999179,991525,990776,990956,997420,993453,996328,991257,994282,999091,997942,994193,993931,990705,993621,994119,995159,992441,994092,992685,991723,994351,994035,994835,996055\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 351\n", + "Returned IDs: 994964,995841,998445,990585,995372,997796,999026,998921,998797,991498,990925,993484,997320,990146,996001,998029,997301,994978,993845,997913,991444,996053,996959,992147,993579,995322,994584,999009,994776,998302\n", + "Ground Truth: 990936,993648,994964,995841,998445,990585,995372,997876,997796,999026,998921,998797,991498,990925,991582,990801,993484,997320,993402,990146,993663,993853,996001,998029,997301,994978,999533,993845,997913,991444\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 352\n", + "Returned IDs: 991498,993484,994503,997597,999005,999263,995372,997320,997301,995841,996560,992637,998634,991743,990829,990936,995034,998779,998839,990585,998445,999658,997305,997908,991626,995802,993443,995771,992811,997470\n", + "Ground Truth: 991498,993484,994503,997597,999005,999263,995372,997320,997301,995841,994917,996560,992637,998849,998966,998634,991743,990829,990936,995034,994183,998779,998839,990585,998445,992510,999658,997305,997908,994611\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 353\n", + "Returned IDs: 993237,998244,992252,998734,995515,998782,999453,990043,998837,990142,991999,994033,991307,990965,992835,993091,999793,996438,991485,993708,991840,992169,996686,996840,995443,991392,998363,993582,999620,991687\n", + "Ground Truth: 993237,998244,994439,992252,998734,995515,992342,998782,999453,990043,998837,990142,991999,994144,994033,991307,994785,993859,990965,992835,991149,993091,999793,996438,991609,991485,993708,991840,996686,992169\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 354\n", + "Returned IDs: 995097,993588,994145,994830,998440,991081,990622,993585,990016,994925,991921,990536,993342,998030,996858,993101,998710,994513,996974,992664,998903,990497,991075,998097,994689,995906,992201,997230,994682,990700\n", + "Ground Truth: 995097,993588,994145,994830,998440,991081,990622,993585,990016,994925,991921,990536,993342,994720,998030,996858,991087,993101,996147,998710,994513,996974,992664,998903,992174,994523,990497,991075,998097,994689\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 355\n", + "Returned IDs: 992169,995080,993143,997309,997918,991743,996918,991698,997002,995783,992558,990479,990561,990882,991257,994972,990897,996867,994334,998180,997586,993717,992944,994577,997822,999676,998726,995516,994409,993260\n", + "Ground Truth: 992169,995080,993143,997309,991052,997918,991743,996918,991698,997002,995783,992558,990479,990561,998045,995401,990882,991257,994972,991061,997723,990897,996867,994334,992933,998180,991447,997586,993717,992944\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 356\n", + "Returned IDs: 999587,992884,993289,999453,996657,997554,996001,990845,997002,990399,996810,992979,996028,994946,995401,996147,993032,996368,995216,992521,996759,997023,999827,994809,992558,998495,992862,999099,992395,999977\n", + "Ground Truth: 999587,992884,993289,999453,996657,997554,996001,990845,994531,997002,990399,996810,992979,996028,994946,995401,996147,993032,996368,995216,992521,995281,996759,997023,999827,994809,993280,992558,990956,998495\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 357\n", + "Returned IDs: 993809,998225,998302,994879,999444,997906,995638,990866,992748,996805,998950,997913,998123,996230,996075,999977,996811,995672,997320,996001,998580,995701,998697,992394,990096,990083,996761,994640,990949,993080\n", + "Ground Truth: 991063,993809,998225,998302,994879,999444,997906,995638,990866,992748,996805,998950,997913,998123,996230,996075,991927,999977,996811,995672,995755,997320,999831,993821,996001,998580,995701,998697,991311,992394\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 358\n", + "Returned IDs: 995580,991677,999333,999301,992668,999414,994464,996192,998170,992134,997913,998216,993222,997818,997192,990957,998600,997106,995870,997841,994748,992555,997071,997437,999624,992696,993853,993791,994197,996296\n", + "Ground Truth: 995580,991677,995775,999333,995266,999301,992668,999414,996499,998599,994464,996192,998170,992134,997913,998216,993222,992946,994904,997818,997192,991081,998152,990957,991422,995014,998686,990647,998600,997106\n", + "Recall: 0.6000\n", + "\n", + "Query ID: 359\n", + "Returned IDs: 990323,991037,999748,991596,995915,995385,990194,999576,995712,996702,997242,994955,995562,990148,994200,990096,997666,994231,993866,993458,999384,995655,990596,990876,999663,997685,999555,991640,991732,991093\n", + "Ground Truth: 990323,991037,999748,991596,995915,995385,990194,999576,995712,996702,997242,994955,995562,994020,990148,994200,995349,990096,997666,993179,994231,999435,998583,993866,992571,993458,999384,995655,990596,996424\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 360\n", + "Returned IDs: \n", + "Ground Truth: 995559,993695,991179,993038,995560,995946,998790,999898,996210,990435,991923,997920,992944,992047,998416,996042,995107,997586,993055,996370,990096,995720,995332,999430,994815,995401,993241,998349,998950,993324\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 361\n", + "Returned IDs: 995891,991544,998651,995946,993561,994413,996472,990096,991958,993738,993566,992448,991247,997252,997386,998545,998451,994285,995536,994365,995423,992631,995146,999673,994409,992801,998317,998703,995547,996733\n", + "Ground Truth: 995891,991544,998651,995946,993561,994413,996472,990096,991958,998973,993738,993566,992448,992260,991247,997252,997386,998545,998451,994285,995536,991447,994365,995423,995303,992631,995146,993433,999543,999673\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 362\n", + "Returned IDs: 994589,991666,995401,997723,991630,996659,990517,998530,999977,995701,996379,996864,998350,998874,991603,998990,992367,998045,999000,998536,994177,995994,994409,992790,992521,994946,999228,999277,991403,997658\n", + "Ground Truth: 994589,991666,995401,992933,997723,991630,996659,990517,998530,999977,995701,996379,996864,998350,998874,991603,999461,998990,992367,998045,999000,998536,994177,995994,994409,992977,992790,991151,992521,994946\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 363\n", + "Returned IDs: 994742,993080,995401,995932,999526,990096,996001,990289,998097,992052,996918,990468,993260,997168,996043,997320,992636,996387,996408,991245,992571,993637,997790,995740,997897,995536,998021,990814,991629,990392\n", + "Ground Truth: 994742,993080,995401,995932,999526,993719,990096,996001,990289,998097,992052,996918,990468,993260,990930,996516,997168,996043,994972,997320,992636,996387,993324,997103,996408,991245,992571,993637,997790,995203\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 364\n", + "Returned IDs: 996209,990423,991484,999555,994181,992234,998861,999576,997850,997666,995780,992716,998006,997242,997685,990194,992209,994335,992892,995209,992768,999273,996809,990167,993508,996386,999319,990154,998122,998598\n", + "Ground Truth: 996209,990423,991484,999555,994181,991003,992234,998861,999576,993615,997850,997666,995780,992716,998006,997242,997685,990194,992209,994335,992892,995209,992768,999273,996809,990167,993508,996386,999319,998419\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 365\n", + "Returned IDs: 994415,992041,993785,997320,996075,990914,992635,997550,996741,993741,991662,995596,995372,996875,992601,997661,996805,993447,997286,992119,997836,993125,997470,999982,997305,999701,991650,990697,991503,999473\n", + "Ground Truth: 994415,992041,993785,997320,996075,990914,992635,997550,996741,993741,991662,995596,995372,996875,995622,992601,995995,997661,996805,993447,997286,992942,992119,997836,993125,997470,999982,997305,999701,991650\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 366\n", + "Returned IDs: 995633,999676,997374,994317,992289,997319,994529,995289,990903,991743,996867,990126,995268,998221,995274,990432,994878,999453,994409,999587,993920,995020,998913,994850,994343,990435,998862,999801,996411,999026\n", + "Ground Truth: 995633,999676,997374,994317,992289,997319,995401,994529,995289,990903,991743,996867,990126,993412,995268,998221,995274,990432,993402,997723,994878,999453,994409,999587,993920,995020,998913,994850,994343,990435\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 367\n", + "Returned IDs: 994964,999151,990545,996219,995961,999331,999261,999367,990922,994896,999210,991382,993877,997720,993215,994705,995927,991378,994782,992446,990210,991751,996990,990957,996805,995861,996949,994783,998495,992114\n", + "Ground Truth: 994964,999151,990545,996219,995961,999331,991345,999261,994843,999352,999367,990922,994896,991921,994577,999210,991382,993238,992942,993877,998294,993154,997720,993215,994705,995927,991378,994782,990210,992446\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 368\n", + "Returned IDs: 995166,993857,998982,994689,991075,997208,996028,991585,995456,998427,993725,998182,990328,994307,995823,990097,993053,994765,995633,992480,996506,993906,995295,997309,995289,990811,992558,998180,991315,998105\n", + "Ground Truth: 995166,993857,998982,994689,991075,997208,996028,992374,991585,995456,998427,993725,998182,990328,994307,995823,990097,993053,995401,994765,995633,992480,996506,993906,995295,997309,995289,990811,992558,998180\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 369\n", + "Returned IDs: \n", + "Ground Truth: 995560,992944,995375,996370,996210,994972,991994,995372,995841,996001,998950,995559,995107,997320,999009,998567,990949,997155,995873,990930,991052,995536,990538,995641,997305,995332,998697,994081,992521,998849\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 370\n", + "Returned IDs: 997320,993484,997913,994604,996143,993154,998967,999948,995802,992442,990066,997470,998170,999658,990532,992119,995372,992330,990949,995865,999851,992546,993157,996075,998808,992896,999535,995351,990585,994363\n", + "Ground Truth: 997320,993484,997913,994604,996143,993154,998967,999948,995802,995228,992442,990066,997470,998170,999658,990532,992119,995372,992330,990949,995865,999851,992546,993157,996075,995942,998808,992896,999535,995351\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 371\n", + "Returned IDs: \n", + "Ground Truth: 991975,993641,990429,995168,992110,999176,995231,993629,998858,993381,995780,997078,991455,996623,991332,992365,991385,993560,999555,991987,996571,994413,990976,994834,992926,997546,990449,998731,993191,998149\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 372\n", + "Returned IDs: 990249,993811,992806,995967,993727,997195,992121,999240,999479,993121,993583,990121,996750,993473,992558,994163,999400,997897,998820,997918,993158,990117,995626,997124,994134,990395,999098,993839,994957,995309\n", + "Ground Truth: 990249,993811,992806,995967,993727,997195,992121,999240,999479,993121,993583,990121,996750,993473,992558,999887,994163,999400,997897,998493,992407,998820,991447,997918,998559,993158,993911,992864,990117,995626\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 373\n", + "Returned IDs: 992225,996089,993174,999041,999216,995005,998200,991984,993834,996230,990051,997258,992167,999219,993928,998934,992213,994902,994167,993322,994248,998989,993489,996657,993766,990472,995516,992238,991157,990547\n", + "Ground Truth: 992225,996089,993174,999041,999216,995005,998200,991984,993834,996230,990051,997258,992167,999219,993928,998934,992213,994902,994167,993322,994248,998989,993489,996657,993087,993766,990472,995516,992238,991157\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 374\n", + "Returned IDs: 998593,995421,991212,992437,993715,997897,991245,997790,995167,996001,990255,996043,998250,994743,993931,998037,998934,997453,990364,995945,996213,995126,991725,994925,996657,995873,993289,990314,991958,990811\n", + "Ground Truth: 998593,995421,991212,992437,993715,997897,991245,997790,995167,996001,990255,996043,998250,994743,993931,998037,998934,997453,995740,990364,995945,996213,995126,991725,994925,996657,995873,993289,990314,991958\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 375\n", + "Returned IDs: 998921,994964,996504,997984,994415,991720,996143,997320,997470,993518,994584,990958,999343,991582,993253,999647,990146,999676,997192,991498,990648,992147,998550,999102,995515,995336,995181,992942,993484,997305\n", + "Ground Truth: 998921,994889,994964,996504,997984,994415,991720,996143,997320,997470,993518,994584,995865,990958,998607,999343,991582,993253,999647,990146,999676,997192,991498,990648,990917,992147,998550,999102,995515,995336\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 376\n", + "Returned IDs: 995372,998849,992944,993241,990585,995375,996809,990489,999544,990226,995560,995841,999711,991498,999210,996318,997320,994972,994402,991994,993388,996001,990043,995641,997436,997113,995443,990532,999641,997002\n", + "Ground Truth: 995372,998849,992944,993241,990585,990060,995375,996809,990489,999544,998567,990226,995560,995841,999711,991498,999210,996318,997320,995401,994972,993533,994402,991994,997848,993388,996001,990043,995641,990894\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 377\n", + "Returned IDs: 998710,999377,994663,994307,998105,996132,994513,997434,993053,994830,999430,994185,999472,999613,996906,994742,992201,999500,994925,996147,992079,996411,994145,999064,991128,990435,992230,995852,991409,999302\n", + "Ground Truth: 998710,999377,994663,994307,998105,996132,995310,994513,990016,997434,993053,994830,999430,994185,999472,999613,990012,996906,994742,992201,999500,994925,996147,992079,996411,994145,999064,991128,990435,992230\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 378\n", + "Returned IDs: \n", + "Ground Truth: 995655,993948,991667,992517,998861,997145,995536,992595,990154,998536,990005,990330,993001,998122,998495,991610,995835,992561,995780,997884,996623,991490,997265,997242,997319,999555,998006,992110,999711,990956\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 379\n", + "Returned IDs: 992532,995974,998686,998334,991480,999451,992349,992645,999711,993976,995314,998417,998079,992383,997521,990383,996196,990401,996262,991015,991208,990376,997758,998061,999639,999913,991977,992718,998118,990908\n", + "Ground Truth: 992532,995974,998686,998334,991480,999451,995560,992349,992645,996894,993174,999711,993976,995314,998417,998079,992654,992383,997521,990383,996196,990401,998849,996262,991015,999216,991208,990376,997758,998061\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 380\n", + "Returned IDs: 997683,996393,998918,991309,994393,990657,993749,992623,992638,997110,997568,990361,998257,995044,998858,999156,993795,999555,995829,999312,995373,991204,998277,997114,990005,994901,993076,991987,992587,992861\n", + "Ground Truth: 997683,996393,998918,991309,994393,990657,993749,992623,992638,997110,997568,990361,998257,995044,998858,999156,993795,999555,991188,990554,995829,997030,999312,995373,997429,991204,998277,997114,991052,990005\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 381\n", + "Returned IDs: 995482,999179,990157,994806,997997,993466,995106,991664,995055,998256,996633,993731,990083,994166,997411,993820,998748,995778,995777,996328,993111,998543,998221,999759,990806,992364,990987,992285,995996,993931\n", + "Ground Truth: 995482,999179,990157,994806,997997,993466,995106,991664,995055,998256,996633,993731,996283,991061,990083,994166,997411,993820,998748,996386,995778,995777,996328,993111,998543,992563,998221,999759,990806,992364\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 382\n", + "Returned IDs: 996147,992161,990771,995167,996001,997790,997918,999898,991667,992950,993289,996657,997554,999150,990399,999099,999453,991212,994266,993143,991585,991485,992384,996549,990314,998720,993280,993157,995536,992367\n", + "Ground Truth: 996147,992161,990771,995167,996001,997790,997918,999898,991667,992950,993289,996657,997554,998215,999150,990399,999099,991311,999453,991212,994266,995855,993143,997163,991585,991485,992384,996549,995281,990314\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 383\n", + "Returned IDs: 992558,990538,998950,990279,999639,996196,997775,997918,997258,994163,992615,995552,999206,999860,996609,990273,997859,999564,996592,998411,991522,994280,995340,994742,995372,999715,992436,991498,991791,993077\n", + "Ground Truth: 992558,990538,998950,990279,999639,996196,997775,997918,995693,997258,994163,992615,995552,999206,999860,996609,990273,997859,999564,996592,998411,991522,994280,995340,994742,995372,999715,992436,991498,991791\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 384\n", + "Returned IDs: 998748,991981,991185,995083,993209,993920,990593,992595,993466,992450,995633,990887,996328,991585,990331,997374,995268,997710,990157,990126,990435,997185,994541,994742,994185,993997,995996,995106,998055,991044\n", + "Ground Truth: 998748,992091,991981,991185,995083,993209,993920,990593,992595,993466,992450,995633,990887,996328,990226,991585,990331,997374,995268,997710,990157,990126,990435,997185,994541,994472,994742,994185,993997,995996\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 385\n", + "Returned IDs: 997852,999077,999676,992169,994922,991334,996325,997204,992612,990432,993260,997822,993717,991840,991743,994964,992813,999453,998950,993355,994742,994318,998779,991435,993237,998123,997309,997918,997379,993143\n", + "Ground Truth: 997852,997476,999077,999676,992169,994922,991334,990490,996325,997204,992612,999916,990432,993260,997822,997496,993717,998041,993673,991840,991743,994964,992813,999453,998950,994044,999843,997135,993355,990214\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 386\n", + "Returned IDs: 998130,991982,993245,995449,993948,990005,993628,998758,991987,991630,993055,995516,998861,998959,994125,991530,997884,990897,995536,995780,992719,990991,996918,991052,992293,992862,995432,998623,998067,995184\n", + "Ground Truth: 998130,991613,991982,993245,995449,993948,990005,993628,998758,991987,991630,993055,995516,998861,998959,994125,991530,998383,997884,990897,995536,995780,992719,990991,996918,991052,992293,992862,995432,998586\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 387\n", + "Returned IDs: 993725,992424,990528,997927,991565,997079,995551,993705,999751,996928,992753,999922,996633,998348,995274,996411,993623,990392,992226,995435,993053,999083,999740,992680,995744,992636,994783,994878,993585,994730\n", + "Ground Truth: 993725,992424,990528,997927,991565,997079,995551,990315,997207,993705,998574,999751,996928,992753,999922,996633,998348,995274,996411,997714,993623,990392,992226,995435,993053,999083,999740,992680,995744,992636\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 388\n", + "Returned IDs: 994578,993856,996867,992448,994140,990751,995289,992239,991790,997319,999235,999474,995633,994409,999875,992364,998910,996368,993293,990814,997723,997046,995640,992289,995030,996277,990940,992521,997868,997466\n", + "Ground Truth: 994578,993856,996867,992448,992468,994140,990751,995289,994419,992239,991790,997319,999235,999474,995633,994409,999875,992364,998999,991185,998910,996368,993293,990814,997723,997046,995640,992289,995544,993280\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 389\n", + "Returned IDs: 990742,991593,991662,991016,997826,998899,999701,997913,995748,991502,998003,999374,991616,997320,995928,998170,994964,991479,993885,999948,993606,992978,991743,998663,997470,991528,990650,991999,999263,995771\n", + "Ground Truth: 990742,991593,991662,991016,997826,998899,999701,997913,995748,991502,998003,999374,991616,997320,999390,995928,990902,991237,998170,993238,994964,991479,993885,999948,993606,998294,992978,991743,990816,998663\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 390\n", + "Returned IDs: 990249,996609,998820,991248,995124,992558,993433,999240,996762,994457,995289,995377,997822,996606,992469,995354,995401,995119,991330,991449,998874,996750,993647,995766,991334,999400,993561,993673,993831,997374\n", + "Ground Truth: 990249,996609,998820,991248,995124,992558,993433,999240,996762,991350,994457,991266,995289,995377,997822,996606,993412,995332,996294,992469,995354,997723,999646,995401,995119,997498,991330,991449,998874,999898\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 391\n", + "Returned IDs: 992052,995873,994742,991451,994689,995295,993738,997897,996580,998256,990930,996328,992445,998623,992239,994280,990956,997329,993504,998045,991185,994255,996277,996001,996028,995823,990005,992636,994307,990096\n", + "Ground Truth: 992052,993456,998427,995873,994742,991451,994689,995295,993738,997897,995408,996580,998256,990930,996328,992445,991255,998623,992239,994280,990956,997329,993504,998045,991185,997918,994255,996277,996001,996028\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 392\n", + "Returned IDs: 996986,990951,992636,996408,999334,991393,990173,996183,997168,999791,992372,993917,998348,996728,993001,998076,999138,997838,998440,993715,996882,994351,992664,999435,992282,999373,994742,990918,997732,996062\n", + "Ground Truth: 996986,990951,992636,996408,999334,991393,990173,990027,996183,997168,999791,992372,993917,998348,996728,993001,998076,999138,997838,998440,993715,996882,998389,990712,994351,992664,999435,992282,999373,994742\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 393\n", + "Returned IDs: 990487,999774,999191,996194,998528,997770,991977,994254,997251,996042,999179,990102,997115,990678,992558,991447,992979,994387,992086,998104,991303,994280,993275,997918,991061,998703,997126,990249,991334,996429\n", + "Ground Truth: 990487,996451,999774,999191,996194,991720,998528,999343,997770,991977,994254,997251,996042,999179,990102,998968,997115,990678,992558,991447,992979,996658,994387,992086,998104,991303,990705,994280,993275,991743\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 394\n", + "Returned IDs: 990535,999144,994217,993168,992630,998951,993083,997015,992331,995336,997320,999437,997082,994964,990925,992748,993389,995252,997002,996001,999019,993484,994757,995824,994854,998090,990958,990231,992384,993050\n", + "Ground Truth: 990535,999144,994217,993168,992630,998951,993083,997015,992331,995336,999916,997320,995947,999437,997082,994964,990925,992748,993389,995252,997002,995216,996001,999019,993484,994757,994916,995824,994854,998090\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 395\n", + "Returned IDs: \n", + "Ground Truth: 994280,991334,997023,995968,990214,995377,995401,999701,991923,994835,995820,991743,998374,990921,995886,994655,992165,992582,995824,994982,992521,998302,992384,993561,997918,991269,994496,996762,995336,998295\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 396\n", + "Returned IDs: 996915,996001,996147,998697,997506,995536,997227,993157,998720,998303,995250,990771,993280,996657,991332,996368,993289,993561,990532,990096,991530,998060,999639,992950,995562,993055,995899,997554,990399,999453\n", + "Ground Truth: 996915,996001,996147,998697,997506,995536,997227,993157,998720,998303,995250,995401,990771,991585,993280,996657,991332,996368,993289,993561,999587,990532,990096,991530,998060,999639,992950,995562,993055,995899\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 397\n", + "Returned IDs: 996178,990532,996805,998950,990096,991981,997320,992318,994307,994742,997906,999444,993809,994185,990843,990364,990083,995401,993080,997913,997790,993342,993504,994416,996397,990814,996370,993621,991459,995438\n", + "Ground Truth: 996178,990532,996805,998950,990096,991981,997320,992318,994307,994742,997906,999444,993809,994185,990843,990364,990083,995401,992636,993080,997913,997790,998365,993342,993504,994416,996397,999647,990814,993834\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 399\n", + "Returned IDs: 998908,998921,997539,997301,993581,995154,995910,993143,999219,990760,996014,991157,993364,997370,990561,996040,998029,998726,997371,993250,998894,991392,997218,997984,992734,990053,991782,990146,992546,994675\n", + "Ground Truth: 998908,998921,994244,997539,997301,999995,993581,995154,990702,999426,995910,993143,999219,992461,998539,990760,996014,995554,991157,993900,993364,997158,993662,997777,997370,990561,992124,999358,996040,993005\n", + "Recall: 0.5333\n", + "\n", + "Query ID: 400\n", + "Returned IDs: 996042,995401,992685,997822,997723,996867,993433,997232,995289,998080,992169,995200,992558,997918,997675,993407,998913,998495,995105,990214,998528,990119,997466,995685,992582,996734,996903,996809,992604,993717\n", + "Ground Truth: 996042,991052,995401,992685,997822,997723,996867,993433,997232,995289,993412,998080,992169,995200,992558,997918,997675,993407,998913,998495,995105,990214,998528,990119,997466,995685,992582,996038,996734,996903\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 401\n", + "Returned IDs: 992978,991883,997320,990210,993431,994964,991721,997913,996904,990114,997470,995841,991249,999167,998550,992944,992546,998364,993072,997371,998365,991801,998540,992692,990096,993662,992310,990925,996143,992066\n", + "Ground Truth: 992978,991883,997320,990210,993431,994598,994964,991721,997913,996904,990114,997470,995841,991249,999167,998550,992114,992944,992546,998364,993072,998698,997371,998365,991801,998540,992692,990096,993662,992909\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 402\n", + "Returned IDs: 997643,994724,997210,992638,995044,997683,990657,997589,999156,999117,998918,992623,993076,998257,996393,998277,993795,994660,993666,991797,999808,995207,991309,998787,991204,990361,998383,995056,992926,991038\n", + "Ground Truth: 997643,994724,997210,992638,995044,997683,990657,997589,999156,999117,998918,992206,992623,993076,998257,996393,998277,993795,994660,993666,991797,999808,995207,991309,990167,998787,991204,994917,990361,998383\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 403\n", + "Returned IDs: 998149,990658,994869,999663,991038,999834,993055,999736,997506,999139,995780,994467,995237,991169,999555,992421,998831,993629,995516,996406,999896,996623,999806,995755,998720,991154,990292,997650,998586,998758\n", + "Ground Truth: 998149,990658,994869,999663,991038,999834,993055,994155,999736,997506,999139,998858,995780,994467,995237,991169,999555,992421,998831,993629,995516,996406,999896,996623,999806,995755,998383,998720,991154,991835\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 404\n", + "Returned IDs: 992493,995807,999826,997189,994677,994301,998820,993530,997232,993223,997420,996139,997184,995625,991051,998316,995521,993149,997075,998080,999475,997675,997634,997574,997061,997824,999961,995493,996009,992801\n", + "Ground Truth: 992493,995807,999826,997189,994677,994301,998820,993530,997232,993223,997420,996139,997184,995625,991051,998316,995521,993149,994141,997075,998080,999475,997675,997634,997574,997061,997824,992683,996903,995366\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 405\n", + "Returned IDs: 998999,995940,997662,993103,995289,996302,999957,991603,995401,995200,999875,990940,997630,996769,994409,997723,995182,996388,990033,997020,998530,994161,994948,991954,997319,998765,993144,995648,992619,998162\n", + "Ground Truth: 998999,995940,997662,993103,995289,996302,999957,997042,991603,995401,996864,995200,999875,990940,997630,996769,994409,997723,995182,996388,990033,997020,998530,994161,994948,991954,997319,998765,995701,993856\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 406\n", + "Returned IDs: 994577,992068,998816,991015,999806,999194,997379,995039,999122,999432,992423,993012,991157,991987,998568,998950,998858,993143,997466,990991,998437,998257,991713,999568,992330,991844,997320,991571,997918,996262\n", + "Ground Truth: 994577,992068,998816,991015,999806,999194,997379,995039,999122,993628,995453,999432,992423,993012,991157,991987,996281,998568,998950,998858,993143,997466,990991,998437,998257,999267,991713,999568,992330,990479\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 407\n", + "Returned IDs: 994717,994809,990255,990361,995401,994092,992293,996657,999629,998593,996809,995701,996759,990573,994431,996930,995289,992558,993848,992367,998536,997897,998990,996393,998250,996777,995963,997822,995873,995864\n", + "Ground Truth: 994717,994809,990255,990361,997088,995401,999587,993515,994092,992293,996657,999629,998593,996043,996809,995701,996759,990573,994431,996930,995289,992558,993848,997411,992367,998536,997897,998990,996393,998250\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 408\n", + "Returned IDs: 994794,993465,990242,993412,998297,999923,991603,997083,995269,998999,990385,990940,995208,991028,991790,997045,995702,995274,997723,995940,999374,999932,997787,998779,991350,997020,997832,995373,996379,991543\n", + "Ground Truth: 994794,993465,990242,994702,993412,998297,999923,991603,997083,995269,998999,990385,990940,995208,997473,991028,991790,997045,995702,995274,997723,995940,999374,999932,997787,998779,991350,997020,997832,995373\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 409\n", + "Returned IDs: 996690,993363,996963,991772,991157,991550,993370,993411,993706,999393,992383,994163,996196,999639,997775,998697,990159,996926,995013,992349,990279,992770,999908,992423,992738,997225,997258,998142,995599,999041\n", + "Ground Truth: 996690,993363,996963,991772,991157,991550,993370,993411,993706,999393,992383,994163,996196,999639,997775,998697,990159,996926,995013,992349,990279,992770,991706,999908,992423,992738,997225,997258,998142,995599\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 410\n", + "Returned IDs: 998170,994240,993431,996592,999077,992880,997470,993260,997320,997918,991801,991348,999701,996296,998180,995826,991928,993696,995357,991564,991131,994964,992748,994552,994360,990949,996761,995748,993646,993154\n", + "Ground Truth: 998170,994240,997699,993431,996592,994584,999077,992880,997470,993260,997320,997918,991801,995094,991348,993454,992927,994666,999701,996296,998180,994450,995826,991928,993696,995266,995357,991564,991131,994964\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 411\n", + "Returned IDs: 991593,992069,998899,994399,997320,991774,992255,990957,991662,999290,997470,997301,999077,993662,995826,991479,998090,999602,994604,990910,994290,996296,992978,990650,993034,993402,997371,993606,991743,995898\n", + "Ground Truth: 995266,991593,992069,998899,994399,997320,991774,992255,991580,990957,998967,991662,999290,997470,997301,999077,993662,995826,991479,993852,998090,999602,994604,990910,994290,996296,992978,992946,990650,998801\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 412\n", + "Returned IDs: 992521,997545,993269,997466,996552,991878,991969,993107,992097,994353,998238,993438,991555,993353,998075,998910,995193,994508,995813,998205,995644,992448,994636,994954,997723,991117,998509,999121,990603,995198\n", + "Ground Truth: 992521,997545,993269,997466,996552,991878,991969,993107,992097,994353,998238,993438,991555,998564,993353,998075,998910,998346,995193,994508,991760,997216,995813,998205,998272,995644,998568,992448,994636,994954\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 413\n", + "Returned IDs: 990956,992558,996809,994742,999179,992052,990776,992604,990361,998495,997752,994643,993077,994351,993948,990657,991257,995105,997914,993121,996043,991145,997110,999191,996277,994280,995373,996328,990705,990083\n", + "Ground Truth: 990956,992558,996809,994742,995963,991052,999179,992052,990776,992604,997723,990361,998495,997752,994643,993077,995408,994351,993948,990657,998543,991257,993799,995401,993079,991319,995105,992685,997914,993121\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 415\n", + "Returned IDs: 991643,992554,993325,997685,992209,993055,991724,999744,993965,996889,994834,991738,990096,999622,994420,996733,999319,993744,995385,991243,995402,991093,995736,996209,999771,999079,995796,990160,992237,996817\n", + "Ground Truth: 991643,992554,993325,997685,992209,993055,991724,999744,993965,996889,994834,991738,990096,999622,994420,996733,999319,993744,999273,995385,998419,995402,991243,991093,995736,993132,996209,999771,999079,995796\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 416\n", + "Returned IDs: 992713,994142,994891,991177,993567,995991,990784,990364,999775,998067,998256,996328,990956,991747,996728,995184,993621,994999,994444,992526,994280,998251,992450,998405,991111,996911,998671,990712,997506,993336\n", + "Ground Truth: 992713,994142,994891,991177,993567,995991,990784,990364,999775,998067,998256,996328,999441,990956,991747,996728,995184,993621,994999,991143,997860,994444,992526,994280,998251,992450,998405,991111,996911,993929\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 417\n", + "Returned IDs: 991198,997789,990043,999620,998232,991196,996681,998236,997275,998983,993582,990155,991801,990858,993570,998837,994922,994785,992907,999076,991510,991999,994964,990585,998280,993237,999053,994612,990142,997288\n", + "Ground Truth: 991198,997789,990043,999620,998232,991196,996681,998236,997275,998983,993582,990155,991801,990858,990809,993570,998837,996318,994922,994785,992907,999076,993675,991510,991999,994964,990585,997876,998280,993237\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 418\n", + "Returned IDs: 994106,994203,999731,991253,995216,990193,998205,998255,997059,995873,992521,992162,997554,998527,995813,996001,994488,990081,998910,993951,993738,990775,991163,992448,999402,991394,995079,990603,997928,997176\n", + "Ground Truth: 994106,994203,999731,991253,995216,990193,998205,994773,998255,997059,995873,992521,992162,997554,998527,995813,996001,994488,990081,998910,993951,993738,990775,991163,992448,999402,991394,995079,990603,997928\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 419\n", + "Returned IDs: \n", + "Ground Truth: 993478,994193,997276,999244,996743,997184,996181,993155,991052,996918,991799,990521,994280,997420,993727,996825,991837,995401,998267,992933,999259,997675,994092,992685,995783,996438,992133,993980,999264,994474\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 420\n", + "Returned IDs: 998886,992293,994382,990662,997868,990361,998017,996809,999922,997466,992582,998495,993390,992862,999587,996147,996042,994793,994689,995679,997529,998982,994730,995701,993561,995702,995289,996391,994431,995536\n", + "Ground Truth: 998886,992293,994382,990662,997868,990361,998017,990399,996809,999922,997466,995470,992582,998495,993390,992862,991603,999587,996147,996042,993831,994793,997723,994689,995679,997529,991578,998982,995562,994730\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 421\n", + "Returned IDs: 999374,998779,992685,993412,997529,996219,994382,997822,991958,995702,996379,995401,993096,995685,998045,998380,998495,995289,998123,992169,997723,994964,994057,992351,993355,996959,995633,996721,997002,993089\n", + "Ground Truth: 999374,998779,992685,991052,993412,997529,996219,994382,997822,999900,990337,991958,995702,996379,995401,993096,995685,998045,998380,995994,998495,995289,990701,998123,991743,992169,993725,993186,997723,994964\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 422\n", + "Returned IDs: 996328,999139,992726,999734,991652,998864,995705,994574,997944,993139,991605,990712,993665,996591,995541,995112,993715,991990,991685,996495,991037,994280,997847,999827,996733,990658,994510,992450,995655,997309\n", + "Ground Truth: 996328,999139,992726,999734,991652,998864,995705,994574,997944,993139,991605,990712,993665,996591,995541,995112,993715,991990,995103,991685,996495,991037,991596,994280,997847,999827,993289,993981,996733,990330\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 423\n", + "Returned IDs: 998470,995214,992558,999791,999767,998122,998279,998495,997732,990956,998765,996001,999191,999827,992713,995438,990089,996219,996647,998710,990364,994280,999138,999852,994119,996809,997586,992780,994742,997453\n", + "Ground Truth: 998470,995214,992558,999791,999767,998122,995222,998279,994972,998495,997732,990956,998765,996001,997603,999191,999827,992713,995438,990091,997920,990089,999916,996219,996647,998710,990364,994280,999138,995107\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 424\n", + "Returned IDs: 990811,998593,996147,999173,991000,996657,992326,996028,998878,990771,992727,996001,998934,995158,999740,990573,998250,993391,992289,993813,995289,995200,995182,993848,996302,997007,997753,999868,998999,991958\n", + "Ground Truth: 990811,998593,996147,999173,991000,996657,990130,992326,993483,996028,998878,990771,992727,996001,998934,994838,995158,999740,995864,990573,998250,993391,996864,992289,993813,995289,995200,995182,993848,996302\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 425\n", + "Returned IDs: 990382,999543,994510,993931,999139,990096,998593,998250,996062,991685,995889,994463,991596,995536,998495,997790,995895,990930,996733,994191,991011,998376,996001,996549,999994,995017,999091,991066,990255,997897\n", + "Ground Truth: 990382,999543,994510,993515,993931,999139,990096,998593,998250,996062,991685,995889,994463,991596,995536,998495,994796,997790,995895,990930,996733,994191,991011,998376,996001,996549,999994,995017,993778,999091\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 426\n", + "Returned IDs: 995591,994964,999367,991588,999210,992612,992944,997379,995200,997529,997822,996867,990690,995289,996990,993215,997406,995052,992169,995923,996219,996042,997723,993265,996328,996809,998495,994742,995360,998765\n", + "Ground Truth: 995591,994964,999367,991588,999210,992612,992944,997379,995200,997529,997822,996867,990690,995289,996990,993215,997406,995052,992169,995923,996219,996042,997723,993265,996328,991921,999352,996809,994307,998495\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 427\n", + "Returned IDs: 992862,997586,991490,995701,990786,999806,993809,995536,998708,999653,995455,996082,993637,993470,999444,990167,998200,997506,993834,996370,999087,993515,998697,995203,998536,993055,995672,991530,999216,990449\n", + "Ground Truth: 992862,997586,995005,991490,995701,990786,999806,993809,995536,998708,999653,995455,996082,993637,993470,999444,990167,998200,997506,993834,990271,996370,999087,993515,998697,993504,995203,993821,998536,997577\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 428\n", + "Returned IDs: 993586,996304,997544,993371,991639,996220,993209,990082,993206,998811,999759,995883,995778,990372,997975,990298,996102,994496,999646,997023,991269,999957,999054,992582,991507,997915,990638,994106,998910,997792\n", + "Ground Truth: 993586,996304,997544,993371,991639,996220,993209,990082,993206,998811,996595,999759,995883,995778,990372,997975,990298,996102,994496,999646,997023,991269,999957,999054,992582,991507,997915,990638,994106,999250\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 429\n", + "Returned IDs: 996734,995401,990337,997191,998297,995685,993412,995373,998045,991603,994409,993561,998999,997406,996968,992448,995940,993288,991140,997723,994948,995648,999530,997822,992685,995994,991630,995886,998779,996042\n", + "Ground Truth: 996734,995401,990337,997191,998297,995685,993412,995373,998045,991603,994409,993561,998999,991052,997406,996968,992448,995940,996277,993288,991140,997723,994948,990930,997918,995648,999530,997822,995873,992685\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 430\n", + "Returned IDs: 996219,996328,990214,999179,996811,997442,999500,994387,998256,998968,995438,997420,999560,994783,995562,995130,994233,995895,990039,992237,997945,994742,997485,993100,999430,993560,999139,994402,994280,996494\n", + "Ground Truth: 994956,996219,996328,990214,999179,996811,997442,999500,994387,996149,993727,995332,998256,998968,995438,991212,997420,999560,994783,995562,996071,995130,997002,994233,995895,999352,999540,990039,994206,992237\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 431\n", + "Returned IDs: 994809,997218,992790,991671,996762,991197,996930,993673,992749,992558,990312,992636,994835,995864,998855,993003,992448,998205,996162,996187,990181,993275,995873,997023,996001,993561,995536,998910,994409,998495\n", + "Ground Truth: 994809,997218,995401,992790,990841,991671,996762,991197,991334,996930,993673,992749,992558,990312,992636,991459,994835,995864,992685,998855,993003,992448,998205,996162,998104,996187,990181,993275,995873,996438\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 432\n", + "Returned IDs: 996279,994533,999301,996192,994688,994757,993852,999883,990215,990469,994854,990275,998190,996714,999089,992511,993457,999327,995034,991379,997012,992384,998170,992087,999548,998011,999953,992854,995875,997605\n", + "Ground Truth: 996279,994533,999301,996192,994688,994757,993852,999883,990215,990469,994854,990275,998190,996714,999089,992511,993457,999327,995034,991379,997012,999500,992384,998170,992087,991650,999548,998011,999953,992854\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 433\n", + "Returned IDs: 999820,990561,995327,992979,994906,997910,992790,994946,994673,993289,994819,995968,991074,996425,994982,994170,997218,994334,994188,997002,994922,999977,995875,999026,997380,999587,990215,997605,992884,990916\n", + "Ground Truth: 999820,990561,995327,992979,994906,997910,992790,994946,994673,993289,994819,990304,995968,998011,992047,991074,996425,994982,995401,996714,994170,997218,999715,994334,997396,994188,997002,994922,998210,999977\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 434\n", + "Returned IDs: \n", + "Ground Truth: 992950,997320,996904,993662,998540,999109,992451,998302,997470,993237,994964,997913,997379,992443,992140,996890,993791,998550,999710,999210,998837,991626,993796,994192,990615,992069,991801,993018,991582,997135\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 435\n", + "Returned IDs: 996802,994964,997320,993389,994341,993662,990146,993484,996296,991609,990949,990843,997379,993781,990585,995094,993696,990231,991743,991732,992950,998950,999210,999647,991154,993260,994742,993603,996469,999430\n", + "Ground Truth: 996802,994964,997320,997747,993389,994341,993662,990146,993484,996296,991609,990949,990843,997379,995560,993781,990930,992120,990585,995094,993696,990231,991743,991732,992950,998950,999210,991886,999647,991154\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 436\n", + "Returned IDs: 992237,999752,990806,998405,997392,993196,996839,991723,992827,993335,994472,995630,998055,994335,994101,994960,996016,992372,997527,993501,997442,990173,992892,992285,991418,990440,997997,991672,993460,991061\n", + "Ground Truth: 992237,999752,990806,991961,998405,997392,993196,996839,991723,992827,993335,994472,995630,998055,994335,994101,994960,996016,992372,997527,993501,997442,990173,992892,992285,991418,990440,997997,991672,993460\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 437\n", + "Returned IDs: 994749,999544,997249,995968,997498,999254,994081,997888,995372,990772,990043,995932,991440,993071,999676,996309,991787,991880,999453,995841,997661,996785,998734,994240,995250,991002,996217,993241,994789,993683\n", + "Ground Truth: 994749,999544,994826,997249,995968,997498,999254,994081,997888,995372,990772,990043,995932,991440,993071,999676,996309,996506,991787,998654,994171,992531,991880,999453,995841,997661,996785,998734,994240,994687\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 438\n", + "Returned IDs: 990550,998200,994624,993834,999087,998232,992862,996082,999651,997913,996661,999783,995296,991256,995701,996062,990096,993342,993504,996001,995184,995672,998458,997586,995132,992293,993489,998697,999647,990366\n", + "Ground Truth: 990550,998200,994624,993834,993902,999087,998232,992862,996082,999651,997913,996661,999783,995296,991256,993403,995701,996062,990096,993342,993504,996001,995184,995672,998458,990866,997586,995132,995927,992293\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 439\n", + "Returned IDs: 998536,992110,991644,990449,991620,993001,992282,992595,990399,992832,999827,997506,990154,998067,995655,990942,995326,991667,996328,992444,998625,996408,999994,992862,995030,994380,992761,991747,998623,992793\n", + "Ground Truth: 998536,992110,991644,990449,991620,993001,992282,992595,990399,992832,999827,997506,990154,998067,995655,990942,995326,997723,991667,996328,992444,998625,998861,992318,996408,999994,999555,992862,993993,995030\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 440\n", + "Returned IDs: 999594,992451,993419,990089,998163,993060,994325,995659,999791,996755,993848,994776,992470,993484,996001,996516,993490,997320,998921,996075,995454,994757,997984,992636,993637,996155,995981,994964,995306,999740\n", + "Ground Truth: 999594,992451,993419,990936,992583,990089,998163,999516,993060,994325,995659,999791,996279,990862,996755,993848,994776,992546,994199,995792,992470,999757,993630,993484,996001,996516,993490,992785,997320,998921\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 441\n", + "Returned IDs: \n", + "Ground Truth: 991178,990980,993914,990648,999343,996219,997984,994266,999404,993238,994964,998500,992546,995942,990146,993072,999824,990595,994415,993048,990650,990736,996028,993518,994889,998458,990545,998581,997855,991325\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 442\n", + "Returned IDs: 991856,993960,991202,994497,994604,999260,990224,995135,996854,991733,990958,999374,997470,999398,994922,990980,991593,995959,998921,995365,993640,994266,990660,998500,997192,997842,994584,998660,994819,995260\n", + "Ground Truth: 991856,993960,991202,994497,994604,992323,999260,999261,990224,995135,999633,996854,991733,990958,999374,999793,997470,991889,999398,998190,994922,995947,995259,990980,991593,995959,998921,995365,993640,993570\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 443\n", + "Returned IDs: 992944,998950,993018,997370,997320,995130,996693,995438,996075,993079,997113,995873,996140,990949,991142,996001,998302,998180,996318,993504,990022,995560,993053,996657,991498,995203,997775,994171,999210,996210\n", + "Ground Truth: 992944,998950,993018,997370,997320,995130,996693,995438,996075,993079,997113,995873,996140,990949,991142,996001,998302,998180,996318,993280,993504,990022,995560,993053,992521,996657,991498,995203,997775,994171\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 444\n", + "Returned IDs: 999868,996042,998866,993597,992125,991301,998564,993433,995905,996112,996781,998165,996769,997406,997046,995289,994419,995702,998593,999080,992160,995439,990681,997822,995401,996360,998045,990984,992519,991463\n", + "Ground Truth: 999868,996042,998866,993597,992125,991301,998564,993433,995905,996112,996781,998165,996769,997406,997046,995289,994419,995702,998593,999080,992160,995439,990706,990681,997822,995401,996360,998045,990984,992519\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 445\n", + "Returned IDs: 991266,990352,990751,996368,993561,993673,997918,999099,993289,998999,997002,998186,990517,993275,994664,999977,998593,996001,991958,991332,996379,992828,990930,996759,995701,992703,990035,997189,995679,993813\n", + "Ground Truth: 991266,990352,990751,996368,995421,993561,993673,997918,999099,993289,998999,997002,998186,991603,990517,999303,999866,993275,994664,995097,996028,997077,995274,999977,998593,994959,996001,991958,991332,996379\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 446\n", + "Returned IDs: 992169,996062,993717,997732,994280,999827,996328,998180,997852,997309,995861,991066,993215,993052,992703,995295,992944,992724,996733,997822,993080,997918,994964,996093,992713,990392,997790,990080,997002,998765\n", + "Ground Truth: 992169,996062,993717,997732,994280,999827,996328,998180,997852,998633,996728,997309,995861,991066,993215,993052,992703,990889,995295,992944,992724,996733,996216,997822,993080,993238,997918,994964,996093,992713\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 447\n", + "Returned IDs: 990366,990089,995569,999898,993401,990271,996219,997855,995306,994919,999610,994279,994792,999651,995095,995096,995551,991593,999192,999791,997340,995868,993402,990173,999617,992704,990179,995384,991840,996052\n", + "Ground Truth: 990366,990089,995569,994373,999898,993401,990271,996219,997855,995306,994919,999417,999610,994279,994792,991572,999651,992174,995095,995096,995210,990438,995551,991593,991172,999192,999216,999791,997340,995868\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 448\n", + "Returned IDs: 999658,994611,991088,998966,997320,993253,993484,991593,994249,990890,991895,996838,992546,999167,998246,990936,990949,993694,994964,996219,990210,999247,994573,991155,997372,990545,997470,994542,996143,999948\n", + "Ground Truth: 999658,994611,991088,998966,997320,990444,993253,995942,993484,991593,994249,990890,991895,996838,992546,999167,998246,990936,990949,993694,994964,996219,990210,997699,999247,994573,998287,991155,999991,997372\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 449\n", + "Returned IDs: 999343,990366,994964,995438,990934,995200,998921,996219,997371,997842,990448,990652,997192,999144,998401,999647,997370,996727,993289,997320,997403,993253,998779,995927,996053,997586,996001,995455,994789,996504\n", + "Ground Truth: 999343,990366,994964,995438,990934,995200,998921,997920,996219,997371,997842,992047,995401,990448,990652,997192,999144,990866,998401,999647,993412,997370,996727,993289,994670,995096,997320,997403,993237,993253\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 450\n", + "Returned IDs: 995981,998828,997584,991467,992671,997976,990066,996755,993696,992733,991131,993047,994894,994100,992972,992119,992692,991372,992635,995336,995596,996075,999883,992748,992041,992366,993484,991456,997426,991662\n", + "Ground Truth: 995417,997843,998653,995981,998828,997584,991467,992671,997976,992931,990066,996755,993696,992733,991131,993047,994894,994100,990780,992972,992119,997449,992692,991372,992635,995336,997256,995596,996075,996966\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 451\n", + "Returned IDs: 991392,997320,996247,999453,993300,999806,999639,996744,998232,990772,991984,993904,997777,996001,995250,997258,995515,992238,993484,997913,993848,990957,990352,992429,994081,990949,992384,997305,992750,991002\n", + "Ground Truth: 995855,999680,991392,997320,996247,999453,993300,999806,999639,996744,998232,990772,991984,993904,997777,996463,996001,995250,997258,995515,992238,993484,997380,997913,993848,990957,990352,992429,994081,993003\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 452\n", + "Returned IDs: 991939,993419,997785,995306,995659,998868,998921,997984,999516,996516,990089,990366,997301,997286,991479,998445,990512,991743,997292,998726,993143,997371,994942,992451,990560,994919,998401,991172,990585,998500\n", + "Ground Truth: 991939,993419,997785,995306,995659,998868,998921,997984,999516,996516,990089,990366,997301,997286,991479,991999,999824,998445,990512,991852,991743,997292,998726,993143,997371,997873,994942,991406,992451,996135\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 453\n", + "Returned IDs: 997411,996368,997868,993751,991002,998265,994730,999486,998370,998990,999586,991585,990974,990412,996379,999977,995633,999639,990132,994895,995679,997554,997822,994964,996160,997371,997320,994380,992423,994409\n", + "Ground Truth: 997411,996368,997868,996181,996145,993751,995702,991002,995937,998265,994730,999486,998370,997723,998990,999586,991585,990974,990412,996379,996959,999801,999977,999537,994826,991157,995633,999639,990132,994895\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 454\n", + "Returned IDs: 996328,994273,995315,994327,996633,996495,997193,995122,998067,999225,999528,994998,993167,992726,993466,998864,993825,997026,997847,999980,993308,994113,993936,993328,998518,993127,992576,990887,996170,996194\n", + "Ground Truth: 996328,994273,995315,994327,996633,996495,997193,993089,995122,999391,998067,999225,999528,994998,993167,992726,993466,998864,992091,990987,993825,997026,999980,997847,993308,994113,993936,993328,996277,998518\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 455\n", + "Returned IDs: 991925,996328,996506,991807,990776,997309,996318,995268,992726,996495,993121,994119,995080,992052,997380,997208,994895,994409,999259,995098,991712,993180,990887,996001,997675,998566,992558,996429,991747,999581\n", + "Ground Truth: 991925,996328,996506,991807,990776,997309,996318,998021,995268,990987,992726,991743,996495,993121,994119,995080,992052,997380,997208,995375,994895,994409,999259,995098,991712,993180,990887,997184,996001,997675\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 456\n", + "Returned IDs: \n", + "Ground Truth: 995080,990043,999784,999516,992318,995442,997918,993052,996489,996193,995119,996829,997155,997320,995375,998726,995336,992031,995246,999741,997301,996887,999453,995641,994972,991816,994162,993528,991348,991440\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 457\n", + "Returned IDs: 997024,995372,997470,997320,994618,995841,998951,999109,992451,992978,991498,990191,993241,991650,991999,992228,993788,990275,993853,993579,998764,998195,999929,997984,998921,991545,999317,996685,990957,992942\n", + "Ground Truth: 997024,995372,997470,997320,992789,994618,995841,998951,996143,998445,999109,990936,992451,992978,991362,991498,995422,990191,993241,991650,991999,992228,993788,990275,993853,993579,998764,998195,999929,997984\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 458\n", + "Returned IDs: 990854,991935,992450,995571,998671,996328,994142,998251,990931,992713,995083,991769,998055,999126,995861,999775,996431,996495,990593,997488,990887,991257,990712,990956,996728,991062,999830,992832,998691,994891\n", + "Ground Truth: 990854,991935,992450,995571,998671,996328,991731,994142,998251,990931,992713,995083,991769,998055,991521,999126,995861,999441,999775,996431,996495,992667,990593,997488,990887,991257,990712,990956,990889,991062\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 459\n", + "Returned IDs: 992604,997110,991052,994753,995373,998318,996488,991844,995807,991309,994643,998707,991110,992387,991987,999834,997675,999136,998858,995105,999478,995963,994677,992558,990915,990729,990657,990131,999445,997466\n", + "Ground Truth: 992604,997110,991052,994753,995373,998318,996488,991844,995807,991309,994643,998383,998707,991110,992387,991987,999834,997675,999136,998858,995105,999478,995963,994677,995449,992558,991571,990915,990729,990657\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 460\n", + "Returned IDs: 995942,999358,997468,999263,999075,990595,995771,993364,994964,996218,990560,998211,993492,990146,994741,995502,999291,998964,990958,999374,992968,998644,991593,994281,993484,996592,992401,994100,993250,997301\n", + "Ground Truth: 995942,999358,997468,999263,999075,990595,995771,993364,994964,996218,990560,998211,993492,990146,994741,995865,995502,999291,998964,996869,990958,999374,992968,999701,998644,991593,990886,994281,993484,996592\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 461\n", + "Returned IDs: 997023,991958,997544,995211,990887,992917,996987,993209,992521,993561,996719,995648,994690,990331,996001,998990,995815,998776,992862,994742,995515,998778,993289,996371,994280,996220,991389,991269,993939,994835\n", + "Ground Truth: 997023,991958,997544,995211,995281,990887,992917,996987,993209,992521,993561,996719,997163,995648,995198,998720,994690,995283,990331,996001,998990,995401,995815,998776,995911,992862,994742,997191,996777,991743\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 462\n", + "Returned IDs: 993269,992521,999990,991969,999516,995813,999866,993684,992426,993561,997554,999513,994949,991869,998361,990035,993738,997822,995177,993813,995216,990603,992448,991795,998910,990845,996368,998205,996719,991076\n", + "Ground Truth: 993269,992521,999990,991969,999516,995813,999866,995283,993684,992426,993561,997554,999513,994949,991869,998361,990035,993738,997216,997822,995177,993813,995216,990603,992448,991795,994386,998346,998910,990845\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 463\n", + "Returned IDs: 998507,999832,997070,992827,998405,998006,999179,995191,993121,997442,991723,998671,992652,991421,999933,992237,996428,994196,993501,993335,994510,996216,997160,995421,991596,994541,994869,996328,991672,994467\n", + "Ground Truth: 998507,999832,997070,992827,998405,990712,998006,999435,999179,995191,993121,997442,991723,998671,990039,992652,991421,999933,992237,996428,994196,993501,996775,993335,994510,997693,996216,997160,995421,991596\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 464\n", + "Returned IDs: 996101,991382,993053,996075,996001,994718,993696,994558,997442,991409,990536,998593,992551,998495,995895,993342,993444,996809,996028,993721,999570,997963,997270,990497,999430,991184,995203,993984,996013,990096\n", + "Ground Truth: 996101,991382,993053,996075,996001,994718,993696,994558,997442,991409,990536,998593,992551,998495,995895,993342,993444,996809,996028,993721,999570,997963,997270,990497,999430,991184,995203,997448,993984,996013\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 465\n", + "Returned IDs: 997768,996082,997586,996001,992862,999284,998269,995455,992446,997334,999413,991628,999963,998154,998303,995823,998708,995536,996915,994907,990366,994964,998447,995535,996216,990096,991751,994672,999651,995887\n", + "Ground Truth: 997768,996082,997586,996001,992862,999284,998269,995455,992446,997334,999413,991628,999963,995256,998154,998303,995823,998708,995536,998568,996915,994907,990366,994964,998447,995535,996856,998440,997531,998734\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 466\n", + "Returned IDs: \n", + "Ground Truth: 996042,997251,990624,993453,992483,998703,991110,992052,997766,997234,991447,997420,998758,995464,990963,990776,995516,996232,991977,992558,992487,996814,992147,998968,993948,995373,997309,998221,999529,993433\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 467\n", + "Returned IDs: 996406,992752,993347,995920,995961,993055,995600,996216,999205,999640,993993,992628,995051,993560,997151,996549,996483,999555,992867,998594,992293,993610,998589,998716,999139,996082,998752,995191,995895,998017\n", + "Ground Truth: 996406,992752,993347,995920,997506,995103,995961,993055,995600,996216,999205,999640,993993,992628,995051,993560,997151,996549,996483,999555,992867,998594,992293,993545,995185,993610,998589,998716,999139,996082\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 468\n", + "Returned IDs: 990866,994163,996429,991880,993948,999860,992558,999639,993831,996196,990930,995552,998495,998697,995289,996188,999977,996442,994742,991790,994140,992024,992423,992469,996379,991550,995633,995983,996178,992121\n", + "Ground Truth: 990866,994163,996918,996429,991880,993948,999860,992558,999639,993831,996196,995873,990930,995552,998495,998697,995289,996188,997723,999977,996442,991052,994742,991790,994140,999587,993725,992024,992423,992469\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 469\n", + "Returned IDs: 998697,995894,997586,990096,994972,991713,997320,995536,997002,999840,998303,991530,996062,992521,991332,996918,997918,997246,995080,992571,996368,995672,995560,993504,990538,998568,998260,991157,993071,996001\n", + "Ground Truth: 998697,995894,997586,991052,990096,994972,991713,997320,995536,997002,995740,999840,993280,998303,994040,991530,996062,992521,991332,997790,996918,997918,997246,995080,992571,994334,996368,995672,995560,999234\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 470\n", + "Returned IDs: 999590,996001,995167,992862,993157,998934,996147,993174,997586,990573,998593,997555,990399,997790,996075,998481,998228,996661,994964,995701,991212,996028,995536,995250,999099,994925,992395,997320,998549,995216\n", + "Ground Truth: 999590,996001,997920,995167,992665,992862,993157,998934,996147,994531,993174,997586,990573,998593,998299,997555,990399,997790,996075,998481,998228,996661,999216,993168,994964,995701,991212,996028,991609,997002\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 471\n", + "Returned IDs: 998186,993813,999099,990081,990576,990845,996368,990035,994140,991969,995245,991350,990820,997868,995813,991958,996379,990930,997716,994436,996719,995016,995538,991011,990751,994396,996039,996127,991790,997622\n", + "Ground Truth: 998186,993813,999099,990081,990576,990845,996368,990035,995547,994140,991969,995245,991350,990820,997868,995813,991958,995923,996379,991024,990930,997716,994436,996719,995016,995538,991011,994386,990751,994396\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 472\n", + "Returned IDs: 990557,996318,993572,997918,996429,996001,995455,992749,999898,993885,994188,998401,997113,995203,995932,992161,993504,991212,996210,996866,995596,992047,990409,990181,992466,991498,997305,998849,992790,998495\n", + "Ground Truth: 990557,996318,993572,997918,996429,996001,995455,992749,999898,993885,994188,998401,990448,997113,995203,995932,991571,990519,992161,993504,994832,991212,996210,999715,994155,990772,991311,996866,997395,995596\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 473\n", + "Returned IDs: 999691,991081,998864,998845,990072,993366,996446,999349,993715,996106,998934,999273,992752,994587,993055,997039,998314,995349,991629,999498,992664,998959,992975,995270,991170,995736,995299,999139,993502,994680\n", + "Ground Truth: 999691,991081,998864,998845,997019,990072,993366,996446,999349,993715,996106,998934,999273,995636,992752,994587,995258,993055,997039,998314,995349,991629,999498,992664,998959,999663,992975,995270,994096,991170\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 474\n", + "Returned IDs: 992398,995203,996429,996001,995932,994087,993300,998734,998568,996075,993918,995536,994221,999840,995358,990196,999898,999473,993260,997555,997894,998123,994972,999210,990409,990096,997977,999731,994789,997305\n", + "Ground Truth: 992398,995203,996429,996001,995932,994087,993300,998734,998568,996075,992484,993918,995536,994221,999840,995855,995358,997140,994042,990196,994832,999898,999473,993260,997555,999963,996764,994531,997894,998123\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 475\n", + "Returned IDs: 999384,990096,998495,996809,997918,993696,997906,998950,997320,991571,996805,992944,993018,998861,995438,993260,993215,993154,996623,999210,994661,999390,991801,993504,999009,990843,999320,994964,999831,996370\n", + "Ground Truth: 999384,990096,998495,996809,997918,993696,997906,998950,997320,998383,990210,991571,996805,992944,993018,998861,995438,993260,993154,993215,996623,999210,993744,994661,999390,991801,993504,999009,991743,990843\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 476\n", + "Returned IDs: 991819,992020,994922,994789,996402,995004,998837,997370,990155,995791,993570,990379,995515,998492,998734,990652,990229,998551,990300,995759,990334,999076,994647,990448,993477,992138,998936,990310,991999,999053\n", + "Ground Truth: 991819,992020,994922,994789,996402,995004,998837,997370,990155,995791,993570,990379,995515,998492,998734,990652,990229,999888,998551,990300,995759,990334,999076,994647,990448,993477,992138,998936,990310,991999\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 477\n", + "Returned IDs: 990866,997364,993168,997586,994782,996232,993642,993831,994964,997732,998107,998668,993878,995906,993689,995482,997268,995107,992862,990934,997305,995944,997004,998458,992052,993172,999676,995536,991999,990096\n", + "Ground Truth: 990866,997364,993168,997586,996352,994782,996232,993642,993831,994964,997732,998107,997493,997279,998668,993878,990690,995906,993689,995482,991325,998369,996647,997268,995107,993630,998837,992862,990934,997305\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 478\n", + "Returned IDs: 994925,998228,999729,998039,990845,991789,990820,993906,990856,991233,991958,996873,996001,996368,990811,996298,998593,991685,995923,991200,993637,996237,995899,990930,992408,996062,998549,991125,995679,992862\n", + "Ground Truth: 994925,998228,999729,998534,998039,990845,991789,990820,993906,990856,991233,991958,996873,996001,996368,990811,996298,998593,991685,995923,991200,999590,993637,996237,997920,999009,995899,990930,992408,996062\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 479\n", + "Returned IDs: 998837,995588,991172,999784,991840,990142,999898,997135,999077,991999,993633,997238,990822,993282,994922,990366,997747,999613,990432,994964,996402,993708,992907,997275,990615,994789,990310,998582,993748,991485\n", + "Ground Truth: 998837,995588,991172,996352,999784,991840,990142,999898,997135,999077,991999,993633,997238,990822,993282,994922,990366,997747,999613,990432,994964,996402,993708,992907,997275,990615,994789,990310,998582,993748\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 480\n", + "Returned IDs: 994964,997913,992546,994100,995981,995942,997320,997470,999444,994778,990231,992330,996219,991721,999374,999740,997301,993168,998540,993484,999328,998495,998797,990925,996075,996809,999710,990957,997614,992443\n", + "Ground Truth: 994964,997913,992546,994100,995981,995942,997320,997470,999444,994778,990231,992330,999109,995560,990588,996219,991721,999374,999740,997301,993168,998540,993484,999328,998495,998797,999900,990925,998163,996075\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 481\n", + "Returned IDs: 999860,992558,996609,996196,996523,993839,997002,991334,992411,992161,993583,996606,995618,994418,996389,995482,991977,998679,997864,999715,996127,999540,996882,997914,990249,990061,995332,994280,999711,991426\n", + "Ground Truth: 999860,992558,996609,996196,996523,993839,997002,997436,991334,992411,992161,993583,996606,995618,994418,996389,997003,995482,991977,990701,998679,997864,999240,999715,996429,996127,999540,996882,997457,997914\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 482\n", + "Returned IDs: 998559,990647,996291,998779,997192,990399,991630,996727,997918,992080,995886,992330,995401,995336,995373,998921,991743,994334,998536,997571,998697,998259,993696,992384,995453,994757,995701,993143,993443,998401\n", + "Ground Truth: 998559,990647,996291,998779,998407,995374,997192,990399,991630,996727,997918,992080,995886,992330,995401,995336,995373,998921,991743,994334,998536,997571,998697,998259,993696,992384,995453,994757,995701,999450\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 483\n", + "Returned IDs: 998950,993018,999019,994171,997320,999701,996626,990083,990532,992944,996210,990843,991951,995560,993686,993080,991142,994081,991503,993696,995372,996024,996075,996726,994280,991348,992521,991224,991372,990375\n", + "Ground Truth: 998950,993018,999019,994171,997320,999701,996626,990083,990532,992944,996210,990843,991951,995560,993686,993080,998567,992318,991142,994081,991503,993696,995372,996024,996075,996726,994280,991348,992521,991224\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 484\n", + "Returned IDs: 995633,995289,991463,993725,991989,995401,990033,998106,999739,995766,990392,992160,997822,993813,992595,994419,996867,993762,997155,999665,993561,991585,994742,998536,997141,994057,998070,992289,996001,992558\n", + "Ground Truth: 995633,995289,991463,993725,997682,997319,991989,995401,990562,990033,998106,999739,995766,990392,998574,992160,997822,993813,992595,994419,996867,993762,997155,999665,999052,993412,990930,993561,991585,996254\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 485\n", + "Returned IDs: 991610,992778,997009,990293,999555,999994,992136,992721,990167,995257,993076,991667,990177,992862,994374,996408,991513,991987,998586,997506,997752,993993,999216,997655,995536,998790,997328,993948,997666,996328\n", + "Ground Truth: 991610,992778,997009,990293,999555,999994,992136,992721,990167,995257,993076,991667,990177,992862,994374,998383,996408,991513,991987,998586,999430,994796,997506,990437,997752,991304,991569,993993,999216,996633\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 486\n", + "Returned IDs: 998913,993696,992469,998950,994742,992661,994163,998790,997117,992289,999235,996809,991157,990340,992124,993809,996178,996609,996196,997320,992558,999639,992423,998662,999676,993355,994964,990866,998196,998827\n", + "Ground Truth: 998913,993696,992469,993529,998950,994742,992661,994163,998790,997117,991445,992289,999235,996809,991157,990340,992124,993809,990930,996178,996609,995873,996196,997320,992558,994317,991559,999639,994529,992423\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 487\n", + "Returned IDs: 991858,998906,990033,999450,991603,999037,996048,998045,993144,990940,991463,992519,994409,992125,995444,996388,990984,991958,991747,993813,998165,994140,998999,996719,994057,995702,999665,995052,995289,996482\n", + "Ground Truth: 991858,998906,990033,999450,991603,999037,996048,998045,993144,990940,991463,992519,994409,992125,995444,996388,990984,991958,991747,993813,995864,998165,994140,998999,997187,996719,994057,995702,999665,995052\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 488\n", + "Returned IDs: 997270,990039,996475,998794,996650,997047,992780,990377,992100,993479,995750,999430,999323,996013,996830,999659,993890,999551,990718,991959,993391,992872,998599,992754,993721,991382,999722,994558,997331,996016\n", + "Ground Truth: 997270,990039,996475,998794,996650,997047,992780,990377,992100,993479,995750,999430,999323,996013,996830,999659,997939,993890,999551,990718,991959,993391,992872,998599,992754,993721,991382,999722,994558,997331\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 489\n", + "Returned IDs: 999946,997476,990043,996093,997135,992169,992907,995791,999204,993570,990483,992612,996438,994922,993237,994144,999453,992632,990580,991146,994703,995489,992994,998734,998523,991473,995004,998837,993355,992813\n", + "Ground Truth: 999946,997476,990043,996093,993675,997135,992169,992907,995791,999204,993570,990483,992612,996438,994922,993237,994144,999453,992632,990580,994827,993238,991146,994703,995489,992994,992342,998734,994629,998523\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 490\n", + "Returned IDs: 994384,996140,991550,991817,990279,994732,998697,997739,996380,995983,992558,991458,995552,992661,998950,994341,996926,994171,994163,993686,990068,991791,990383,996196,991713,998827,997258,990456,990343,993018\n", + "Ground Truth: 994384,996140,991550,991817,990279,994732,998697,997739,996380,995983,992558,991458,995552,992661,998950,994341,996926,994171,994163,993686,990068,996894,991791,990383,995372,996196,998567,991713,998827,990769\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 491\n", + "Returned IDs: 990096,998968,993624,994636,992716,996386,999103,997884,999711,999119,996209,999282,998861,990072,990154,999287,995536,997803,991451,999319,993055,993744,999443,991724,996283,999384,996893,999744,994834,992068\n", + "Ground Truth: 990096,998968,998419,993624,994636,992716,996386,991936,999103,991167,997884,999711,999119,996209,999282,998861,990072,990154,999287,995536,997803,991451,999319,993055,999443,993744,991724,996283,996429,999384\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 492\n", + "Returned IDs: 996284,997182,996809,996830,994465,994382,991382,997318,990167,995895,991881,992551,999555,994927,990361,996706,993391,993890,998495,990145,992752,990226,994431,996580,992780,995978,998006,996494,991767,998895\n", + "Ground Truth: 996284,997182,996809,996830,994465,994382,991382,997318,990167,995895,991881,992551,999555,994927,990361,996706,993391,993890,998495,990145,992752,990226,994431,996580,992780,999886,994355,995978,998006,996494\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 493\n", + "Returned IDs: 993283,991485,992309,999639,994042,991157,995562,990399,995516,998232,995443,991015,999806,998180,992124,996744,994809,998697,993260,999041,996368,991984,993055,993289,996657,992862,992905,991002,996138,998758\n", + "Ground Truth: 993283,991485,992309,999639,994042,991157,995562,990399,995516,998232,995443,991015,999806,998180,992124,996744,994809,998697,993260,999041,996368,991984,993055,994922,993289,996657,992147,992862,992905,991002\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 494\n", + "Returned IDs: 992169,996325,997852,992152,997476,992632,992813,994922,993717,996093,999676,990266,994318,999453,994761,996438,992612,997822,994148,999077,997797,994956,990268,991334,998999,998765,993080,998138,998523,990080\n", + "Ground Truth: 992169,996325,997852,992152,997476,992632,992813,992309,994922,993717,996093,990490,999676,990266,994318,999843,999453,994761,996438,992612,997645,990399,997822,994148,994161,999077,995210,997797,998171,994956\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 495\n", + "Returned IDs: 994960,991383,992164,999138,996537,990089,996011,998812,992026,996037,996301,992827,996328,990730,992372,998348,993335,993715,991390,991885,999227,995191,998076,997357,998671,992832,993568,992237,999430,993413\n", + "Ground Truth: 994960,991383,992164,999138,996537,990089,996011,998812,992026,996037,996301,992827,996328,990730,992372,998348,993335,993715,991390,991885,999227,995191,998076,997357,998671,992832,993568,992237,999430,990040\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 496\n", + "Returned IDs: 992124,991984,995516,998697,994742,999136,995701,998758,993186,993727,995672,995562,992747,993834,996075,993342,996657,991530,994082,994809,999783,994879,998950,995203,995981,995119,994280,994325,996884,992480\n", + "Ground Truth: 992124,991984,995516,998697,994742,999136,995701,998758,993186,993727,995672,995562,993470,992747,993834,996075,994374,993342,996657,999041,991530,994082,991052,994809,999783,997377,994879,998586,998950,995203\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 497\n", + "Returned IDs: 997969,993765,993642,997732,999151,994964,990226,996216,993929,990091,994927,993080,998436,998107,998495,998416,997410,997004,993906,991633,997453,990690,996647,990632,997586,990845,992820,993831,996171,998950\n", + "Ground Truth: 997969,993765,993642,997732,999151,994964,997111,990226,996216,995920,993929,990091,994927,990441,993080,998436,998107,998495,998416,997410,997004,993906,991633,997453,997112,990690,996647,990632,991239,990580\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 498\n", + "Returned IDs: \n", + "Ground Truth: 999899,994906,996759,992648,992651,999977,998083,990304,990260,991175,998301,992979,993713,992126,993275,996368,997059,999820,996714,996425,993032,996075,997554,993280,994860,997910,999587,999453,994334,994946\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 499\n", + "Returned IDs: 998080,991334,999479,998820,991025,995190,993931,997395,994222,990126,998493,994204,991449,991743,990733,996609,992685,995401,999098,990249,992604,997515,995289,999788,990131,994431,999961,999533,993566,997209\n", + "Ground Truth: 998080,991334,999479,998820,991025,995190,993473,993931,997395,994222,990126,997030,998493,999586,994204,995612,991449,991743,990733,996609,993929,997212,992685,995401,999098,990249,992604,997515,995332,995289\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 500\n", + "Returned IDs: 999009,991776,995895,990096,993397,997586,990930,990786,999430,995455,994742,996624,990866,990701,990271,997028,996830,997768,999599,991154,995701,996216,998495,996192,995435,999639,995899,996001,997378,999261\n", + "Ground Truth: 999009,991776,995895,990096,993397,997586,990930,990786,999430,995455,994742,996624,990866,990701,990271,997028,996830,997768,999599,991154,995701,996216,998495,996192,995435,999639,995899,991572,996001,999647\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 501\n", + "Returned IDs: 998158,997453,994463,992488,996062,998874,992322,990930,996915,991021,994783,995289,999139,993561,990076,990104,991685,990226,999091,992326,995783,996028,991857,992703,992182,990899,995126,991633,998302,998593\n", + "Ground Truth: 998158,997453,994463,992488,996062,998874,992322,990930,996915,991021,994783,995289,999139,991154,994777,993561,993673,996071,990076,990104,991685,990226,995113,999091,992326,995783,997797,996028,991857,992703\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 503\n", + "Returned IDs: 994894,998353,995981,994100,997913,995942,992228,997946,999243,991045,997301,994266,998090,997728,998246,999374,996219,996805,993186,992384,997320,999331,997614,997304,999444,992848,993529,992124,998417,995701\n", + "Ground Truth: 994894,998353,999599,995981,994100,997913,995942,992228,997946,999243,999658,991045,990701,997301,994266,998090,997728,998246,999374,996219,991468,996805,993186,992384,997320,999331,997614,997304,999444,992848\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 504\n", + "Returned IDs: 990532,999453,995852,994081,994955,993053,995295,996147,992408,994042,995203,995456,990392,990771,991609,999052,998039,997002,999639,996918,997918,993809,998855,993906,996368,996001,992944,998536,996879,993157\n", + "Ground Truth: 990532,999453,995852,994081,995130,994955,993053,995295,996147,992408,994042,995203,995456,990771,990392,991609,999052,998039,997002,999639,996918,995401,997918,993809,998855,993906,994226,996368,992832,992562\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 505\n", + "Returned IDs: 991698,997395,993407,992790,990215,994671,991404,993409,992832,995375,995080,991607,997515,990889,995685,992576,995648,996245,992749,999052,990930,998536,995968,998158,997192,992036,994280,999967,996918,994271\n", + "Ground Truth: 991698,997395,993407,992790,990215,994671,991404,993409,992832,995375,995080,991607,994855,997515,990889,995685,992576,991576,995648,996245,992749,999052,990930,997391,998536,993412,995968,998158,997192,992036\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 507\n", + "Returned IDs: 994472,999791,991921,990089,992322,994402,990091,992820,995720,998107,997939,999876,996355,998355,990226,998630,995535,996761,993775,993584,997349,995638,993929,991751,998169,993378,991382,993637,994927,999294\n", + "Ground Truth: 994472,999791,991921,990089,992322,994402,990091,995222,992820,995720,998107,997939,999876,998765,996355,990622,996050,998355,990226,999642,998630,995535,998594,996761,993775,993584,997349,995638,993929,991325\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 508\n", + "Returned IDs: 992582,991743,990949,991782,990958,995274,999533,997614,994850,994266,995949,995981,999331,990957,997913,990493,991889,999343,990910,998921,998779,998302,997301,994919,997320,991984,995515,994894,997571,995886\n", + "Ground Truth: 992582,991743,990949,991782,990958,995274,995374,999533,993961,997614,994850,990490,994266,995949,995981,997645,991479,996648,999331,990957,997913,990588,990359,990493,991889,999328,993431,999343,990910,998294\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 509\n", + "Returned IDs: 998982,990328,992226,990528,990701,998671,992230,991334,990711,998719,994751,999500,994783,994689,995990,995295,997586,990126,996116,990864,991075,991464,996645,994558,997485,999430,998543,995020,993656,996481\n", + "Ground Truth: 998982,990328,992226,990528,990701,996110,992068,998671,996939,992230,991334,990468,990711,998719,994751,999500,994783,994689,991239,999916,995990,995295,991771,997379,997586,990126,996116,990011,990864,991075\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 510\n", + "Returned IDs: 997913,990650,991593,998500,994964,997301,997320,998170,999982,997192,992978,990066,999948,999167,992087,998899,995802,996219,991335,997470,998808,999647,996504,998550,995961,990210,995949,998302,998921,991479\n", + "Ground Truth: 997913,990650,991593,998500,994964,997301,994831,997320,998170,999982,997192,992978,994618,990066,995942,999948,999167,992087,998899,995802,996219,991335,997470,998808,999647,996504,998550,995961,996053,990210\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 511\n", + "Returned IDs: \n", + "Ground Truth: 993387,994192,998899,996946,992464,992119,997913,996592,990957,991479,990890,997320,993662,992134,993174,995336,994978,999658,996296,996075,993853,992330,991260,999559,998550,998090,991131,994432,999948,996397\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 512\n", + "Returned IDs: 997168,996277,999281,991245,993275,999526,992315,994743,993715,990409,991685,992636,996001,998623,999724,990956,991349,991037,998536,993560,994016,990951,998045,995701,993561,995562,993738,994409,993948,994353\n", + "Ground Truth: 997168,996277,999281,991245,993275,999526,992315,994743,993715,990409,991685,992636,996001,998623,999724,990956,991349,991037,998536,993560,994016,990951,998045,995701,993561,995562,991755,995835,993738,994409\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 513\n", + "Returned IDs: 991375,993906,996216,996082,997790,995991,994925,996062,998950,996599,996178,995107,991769,995895,996001,999827,990364,993561,992780,995705,994280,993931,991142,992847,990096,994742,992322,992680,990930,996196\n", + "Ground Truth: 991375,993906,996216,996082,997790,995991,994925,996062,998950,996599,996178,995107,991769,995895,992473,996001,995961,999827,990786,990364,999009,993561,990886,992780,995705,994280,993931,990887,991142,995586\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 514\n", + "Returned IDs: 991396,990925,998797,990595,997972,999262,996959,995884,994964,997102,993914,996532,990146,998501,997040,999600,994673,993389,997320,993761,990863,991008,993083,991086,990797,992284,992147,996714,999444,993484\n", + "Ground Truth: 991396,994160,990925,998797,990595,997972,999262,996959,995884,994964,997102,993914,995588,996532,990146,998501,993168,997040,997028,999600,994673,993389,997320,995909,993761,997189,990863,991008,993083,991086\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 515\n", + "Returned IDs: 996379,991255,995268,990930,993561,992367,997925,990984,993673,992703,996198,996277,992294,994589,992115,995701,990133,996867,990899,994113,995289,991011,998779,996328,996918,999977,990940,990033,996930,996678\n", + "Ground Truth: 996379,991255,995268,990930,995963,993561,992367,991747,991603,996913,991459,997925,990984,993673,992703,996198,996277,992294,992660,994589,990897,992115,995701,990133,996867,990899,994113,995289,991011,998779\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 516\n", + "Returned IDs: 996187,997868,996605,998941,991349,991795,998527,995814,998255,990732,999121,999926,994409,998205,991878,997998,998796,990035,999731,991163,992858,991969,996618,992311,998143,995813,994638,994773,994140,995856\n", + "Ground Truth: 996187,997868,996605,998941,991349,991795,998527,995814,998255,990732,999121,999926,994409,998205,991878,997998,998796,990035,999731,991163,992858,991969,996618,992311,998143,995813,994638,994773,990481,994140\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 517\n", + "Returned IDs: 996038,998832,995507,997723,995139,997046,996323,995905,999619,997406,996867,992545,993433,999000,998165,993421,999530,995289,992130,997832,996769,997020,996112,991027,993390,995753,992605,995052,997607,996968\n", + "Ground Truth: 996038,998832,995507,997723,995139,997046,996323,995905,999619,997406,996867,992545,993433,999000,998165,993421,999530,995289,992130,997832,996769,997020,996112,991027,993390,996166,992660,996815,995753,992605\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 518\n", + "Returned IDs: 993501,999179,997733,998671,992193,992652,996803,996211,999832,990784,993931,997068,992827,994929,992237,997967,992981,998870,996809,997126,991672,994402,990186,990852,994119,993335,996328,994541,993929,997331\n", + "Ground Truth: 993501,999179,997733,998671,992193,992652,996803,996211,999832,990784,993931,997068,992827,994929,993439,992237,997967,992981,994094,998870,996809,997126,995790,991672,994402,990186,990852,994119,993335,996328\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 519\n", + "Returned IDs: 992315,991037,990596,990834,996562,994302,995562,994123,995349,996286,991648,995500,990630,998358,998052,996908,999435,995740,994955,990203,995366,998855,992199,996884,998303,993504,991212,996001,990409,994743\n", + "Ground Truth: 992315,991037,990596,990834,996562,994302,995562,994123,995349,996286,991648,995500,990630,998358,998052,996908,996677,999435,995740,994955,990203,995366,998855,992199,996884,998303,991956,993504,991212,996001\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 520\n", + "Returned IDs: 997320,997918,993512,992521,998260,995336,999731,993071,994964,993492,994360,999516,996397,994081,998365,994290,995826,992124,995274,997977,992330,992909,991880,998568,992423,995560,996085,994577,998950,998550\n", + "Ground Truth: 992692,997320,997918,993512,990171,992521,998260,995336,999731,993071,994964,993492,994360,999516,996397,994661,994081,998365,998246,994290,995826,992124,996417,995274,997977,992330,992909,991883,991880,992047\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 521\n", + "Returned IDs: 999453,994081,990661,990532,996075,997370,990652,998567,998492,998734,996309,990882,995515,997320,996626,992521,995372,993280,995203,991869,994789,990083,996368,995245,993241,994449,990043,996001,992617,991348\n", + "Ground Truth: 999453,994081,990661,990532,996075,997370,990652,998567,998492,998734,996309,990882,995515,997320,996626,992521,996193,995372,991713,993280,995203,991869,999123,994789,990083,999954,996368,995401,992031,995245\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 522\n", + "Returned IDs: 997822,993433,995289,995633,996042,998106,996494,996867,998165,992160,995401,997046,993390,997766,995040,990252,995200,994742,992558,993053,996769,992521,994783,995052,990435,994529,992448,997961,991301,992576\n", + "Ground Truth: 997822,993433,995289,995633,996042,998106,997797,993857,996494,996867,998165,992160,995401,995873,997046,993390,997918,997766,995040,990252,995200,994742,992558,993053,996769,992521,994783,996028,995052,990435\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 523\n", + "Returned IDs: 998246,992896,990066,997320,993484,994225,996560,998540,998634,999005,995400,992839,997102,998384,997610,995841,998603,996598,997178,994164,994214,991999,994964,991016,994183,991281,994611,999810,994873,994632\n", + "Ground Truth: 998246,992896,990066,997320,993484,994225,996560,999757,998540,999102,998634,999128,995904,995224,999005,992986,995400,994542,992839,991567,997102,998384,997610,995841,998603,990269,996598,997178,994164,994214\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 524\n", + "Returned IDs: \n", + "Ground Truth: 990882,994612,997465,999698,998937,997403,993845,998607,995515,999036,996309,992228,994922,995788,997164,995154,993002,992309,996840,993028,998921,999671,998244,992843,993143,997591,991510,995893,996653,998401\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 525\n", + "Returned IDs: 998225,990022,996075,997305,991348,993809,993341,996626,990697,995203,996001,994416,998359,992635,990532,996538,993721,993696,990083,995250,990573,990786,999289,993637,995899,990866,992557,994233,994081,999453\n", + "Ground Truth: 998225,990022,996075,997305,991348,993809,993341,996626,990697,995203,996001,994416,998359,992635,990532,996538,993721,993696,995613,990083,995250,990573,990786,999289,993637,995899,990866,992557,994233,994081\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 526\n", + "Returned IDs: 998536,995655,995385,996245,991332,995168,992832,991630,995589,994280,995231,996760,993901,991654,998565,995536,992110,996501,999443,993489,992793,997965,993194,997145,993037,994380,998697,997158,998303,999994\n", + "Ground Truth: 998536,995655,997702,995385,996245,991332,995461,995168,992832,991630,995811,995589,994280,993381,995231,996760,993901,991654,990429,998565,995536,992110,996501,999443,993489,992793,997965,993194,993580,997145\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 527\n", + "Returned IDs: 995596,993341,996741,990697,997305,996075,994886,992041,994985,993785,996178,993119,994859,996755,993154,997661,994440,994971,993904,992661,990022,993696,996210,997291,992635,993741,991650,997320,998715,996616\n", + "Ground Truth: 995596,993341,996741,990697,997305,996075,994886,992041,994985,993785,996178,990571,993119,994859,996755,993154,997661,994440,994971,993904,992661,990022,997699,993696,996210,997291,992635,993741,991218,991650\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 528\n", + "Returned IDs: 991795,999800,997868,996187,994140,996867,993003,990341,998205,992448,991969,990845,992994,998633,994419,994409,996042,993561,993813,995289,995864,996169,998361,997089,995702,996719,991790,997909,994386,999250\n", + "Ground Truth: 991795,999800,997868,993412,996187,994140,996867,993003,990341,998205,992448,991969,990845,992994,998633,994419,994409,996042,993561,993813,995289,995864,996169,998361,997089,995702,996719,991790,997909,994386\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 529\n", + "Returned IDs: 992636,997168,995837,997201,995295,993715,994621,999052,998076,995159,994488,997838,991451,990956,996505,994380,998021,990336,990887,990889,996147,995836,993448,996277,992973,998174,995521,991007,997329,992450\n", + "Ground Truth: 992636,997168,995837,997201,995295,993715,994621,999052,998076,995159,994488,997838,991451,990956,992832,992595,996505,994380,998021,990336,990887,990889,996147,995836,993448,996277,992973,998174,995521,991007\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 530\n", + "Returned IDs: 990072,990096,990154,999663,992716,995536,994103,998861,994834,994756,993055,994636,999711,999677,999898,992862,996406,993139,996733,996623,997506,992293,995963,997480,994640,994771,997154,996386,996209,995701\n", + "Ground Truth: 991167,990072,990096,990154,999663,992716,995536,994103,998861,994834,994756,993055,998419,994636,999711,995562,999677,999898,992862,996406,993139,993624,996733,996623,995103,997506,992293,992110,995963,997480\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 531\n", + "Returned IDs: 994749,991673,992685,992165,990772,995886,999182,993840,992677,999453,994655,997433,999586,992052,991943,995274,995401,991787,990432,995932,991585,995968,999820,997370,997574,990957,993260,995515,990103,990214\n", + "Ground Truth: 994749,991673,992685,992165,990772,995886,999182,995374,996949,993840,992677,997498,999453,991110,994655,997433,999586,992087,991052,992052,991943,995274,993673,995401,991787,990432,995932,991585,993412,995968\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 532\n", + "Returned IDs: 996216,990930,999009,998950,994964,995961,999453,993484,997320,995895,999716,993906,999294,993080,999430,990096,997379,994925,991498,998593,997918,990560,992780,999849,996911,998455,994742,993389,995438,998495\n", + "Ground Truth: 996216,990930,999009,998950,994964,995961,999453,993484,997320,995895,998123,999716,993906,999294,993080,999430,992497,990096,997379,994925,990445,991498,998593,997918,990560,992780,999849,996911,998455,992847\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 533\n", + "Returned IDs: 997506,992154,996775,995226,994891,997944,997050,991633,998933,998177,999139,999079,993665,999576,993347,991712,998536,992793,990194,996817,990959,992170,997666,997906,994955,994280,996733,995536,993821,990196\n", + "Ground Truth: 997506,992154,996775,996055,995226,994891,997944,997050,991633,998933,998177,999139,999079,993665,999576,993347,991712,998536,992793,990194,996817,990959,992170,997666,997906,994955,994280,996733,995536,999091\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 534\n", + "Returned IDs: 999876,993484,993504,998302,990092,993803,997320,998294,996869,997370,997913,998029,991498,993741,998580,992384,993885,999535,999374,990146,991043,991593,999982,998246,994089,994366,995949,995372,997661,992733\n", + "Ground Truth: 999876,993484,993504,998302,990092,993803,997320,998294,996869,997370,997913,998029,999363,991498,993741,994826,998580,992384,993885,999535,999374,990146,991043,991593,999982,998246,994089,994366,995949,995372\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 535\n", + "Returned IDs: 994685,994160,996343,990936,998634,990925,993484,998966,990146,992546,999262,998644,998797,995884,993885,990958,997239,997102,991558,996532,993389,992147,999374,991396,993083,994964,996295,998153,990224,996874\n", + "Ground Truth: 994685,994160,996343,990936,994758,998634,990925,997876,993484,998966,990146,992546,999262,999620,998644,998797,995884,993885,990958,997239,997102,991558,996532,993389,992147,999374,991396,993083,994964,996295\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 536\n", + "Returned IDs: \n", + "Ground Truth: 993497,996763,998293,990956,995058,994073,995783,995159,996276,990149,992997,990872,995184,997713,991850,992981,997897,990306,992526,997746,993727,996189,992052,994280,993478,997752,996911,994315,992784,999514\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 537\n", + "Returned IDs: 997958,993767,997185,992272,994742,993997,999235,999279,998733,994730,993209,999402,993856,997566,995268,995633,999805,998162,999160,998962,991585,994140,996867,998748,993466,997660,995640,992289,993751,998898\n", + "Ground Truth: 997958,993767,997185,992272,994742,993997,999235,999279,998733,994730,993488,993209,999402,993856,992063,997566,995268,995633,999805,998162,999160,998962,991255,991585,994140,996867,998748,993466,997660,995640\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 538\n", + "Returned IDs: 990589,999555,990167,992778,992234,994210,992353,996209,995610,991610,990194,994756,997242,998006,998495,992628,999634,994355,997850,993224,995879,998122,999994,998861,996794,991484,995209,997415,990107,998435\n", + "Ground Truth: 990589,999555,990167,992778,992234,994210,992353,996209,995610,991610,990194,994756,997242,998006,998495,996342,992628,999634,993055,994355,997850,993224,995879,998122,999994,998861,999771,996794,992745,994927\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 539\n", + "Returned IDs: 990157,995482,995083,992859,992450,992434,999934,994428,996306,991769,993731,992449,999240,994185,991935,995784,999695,998671,991185,992595,990593,994998,990331,993777,990784,993437,993856,998256,990126,990887\n", + "Ground Truth: 990157,995482,994976,995083,992859,992450,992434,999934,994428,996306,991769,993731,992449,999240,994185,991935,995784,999695,998671,991185,992595,990593,994998,990331,993777,990784,993437,993856,998256,990126\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 540\n", + "Returned IDs: \n", + "Ground Truth: 995300,999981,994742,995633,999474,992468,999372,998913,995012,990905,998765,992415,999788,995494,998543,998450,995289,996302,998557,992289,996057,998895,991590,991185,990126,990392,992598,999235,992239,995030\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 541\n", + "Returned IDs: 998965,996219,994452,992998,995339,990780,999683,995455,996111,994777,991370,996178,997004,995460,990886,996612,998364,991373,990931,990366,993128,999343,998390,998593,999876,991179,998157,996564,995438,995332\n", + "Ground Truth: 998965,996219,994452,992998,995339,990780,999683,995455,996111,994777,991370,996178,992070,997032,997004,995460,994981,990886,996612,998364,991373,990931,990366,993128,999343,997813,993690,998390,999013,997706\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 542\n", + "Returned IDs: 997131,993185,994063,999155,993232,995461,996809,994643,990255,997462,990941,994122,998632,995963,998855,990930,995421,998695,992809,998786,996930,997723,998919,996797,992558,998593,995766,991767,990956,991532\n", + "Ground Truth: 997131,993185,994063,999155,993232,995461,996809,994643,990255,997462,990941,994122,998632,995963,998855,990361,990930,995421,998695,992809,998786,996930,997723,998919,996797,992558,998593,995766,991767,990956\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 543\n", + "Returned IDs: 992944,993018,995560,998950,991142,995438,996726,998160,996140,990375,990843,991650,993324,998364,997320,991951,995107,999210,994171,994964,995130,995332,996370,998697,996653,990585,991206,997002,992318,996805\n", + "Ground Truth: 992944,993018,995560,998950,991142,995438,996726,998160,996140,990375,990843,991650,993324,998364,997320,991951,995107,999210,994171,994964,995130,999647,997379,995332,996370,999109,998697,998180,993499,996653\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 544\n", + "Returned IDs: 993755,992761,991850,998778,991779,992595,995536,994356,992832,999107,999256,993055,995385,996001,996216,998536,990127,995501,999384,991644,995655,996328,999798,992561,997586,997141,991667,994076,992517,990392\n", + "Ground Truth: 993755,992761,991850,998778,991779,995222,997155,992595,995536,994356,992832,999107,999256,993055,995385,996001,996216,998536,992110,997158,990127,995501,991474,999384,991644,995655,996328,992224,999798,992561\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 545\n", + "Returned IDs: 997391,992428,991698,996930,992933,998080,998855,994193,993255,991257,993288,997918,991337,992179,995357,991743,996762,995401,999144,992516,990459,996903,992531,998121,991110,992604,997045,991459,997942,993673\n", + "Ground Truth: 997391,992428,991698,996930,992933,990438,998080,998855,994193,991619,993255,991257,993288,997918,991337,992179,999527,995357,991743,992850,996762,995511,995401,994062,999144,992685,998180,992516,990459,996903\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 546\n", + "Returned IDs: 994419,996745,992894,990252,997776,996322,996148,998593,992660,995643,997046,998486,997940,995289,992125,991301,999482,995923,996867,992048,997822,998045,995702,991958,995063,998564,996323,997937,994173,998165\n", + "Ground Truth: 994419,996745,992894,990252,997776,996322,996148,998593,992660,995643,997046,998486,997940,995289,992125,995470,991301,999482,995923,996867,992048,997822,998045,993412,995702,991958,995063,998564,998105,995401\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 547\n", + "Returned IDs: 991630,992862,990005,999450,998536,994409,996368,992521,991844,998779,997918,991987,999653,991332,995701,995373,993948,995536,991667,995401,996809,995200,996657,994955,996328,991795,993813,998495,994946,995702\n", + "Ground Truth: 991630,992862,990005,999450,998536,998999,994409,996368,992521,991844,998779,997918,991987,999653,991332,995701,993412,995373,993948,998858,998697,991667,995536,997630,995401,996809,995864,995200,996657,994955\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 548\n", + "Returned IDs: 995716,999261,994922,999620,990701,990934,997371,990648,996916,997770,999613,991588,994990,995693,992944,992384,997403,996219,991999,995551,999343,996840,996990,997529,995387,993215,990366,998921,999610,998523\n", + "Ground Truth: 990092,995716,998991,990315,999261,994922,999620,997920,990701,990934,997371,990648,996916,991921,997770,991196,999613,991588,994990,995222,995401,995693,998641,992944,992384,996419,997403,997855,996219,999151\n", + "Recall: 0.6000\n", + "\n", + "Query ID: 549\n", + "Returned IDs: 997897,998250,993483,999463,992437,995421,992727,998593,990255,990521,992367,998934,993175,995626,998628,996181,991245,998162,994280,995401,990844,998032,996001,994809,997918,993727,996657,993232,996043,995823\n", + "Ground Truth: 997897,998250,993483,999463,992437,995421,992727,998593,990255,990521,992367,998934,993175,995626,998628,992407,996181,991245,998162,993489,994280,995182,995401,990844,998032,996001,994809,997918,993727,996657\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 550\n", + "Returned IDs: 997320,991498,995841,995372,990585,993484,992944,996959,993018,997305,994964,996001,990949,997301,998302,998950,997370,995203,990096,993885,998246,998495,998849,995438,997913,995375,992119,997796,996318,990957\n", + "Ground Truth: 997320,991498,995841,995372,990585,993484,992944,996959,993018,997305,998445,994964,996001,990949,991052,997301,998302,998950,997370,995203,990096,993885,991743,998246,998495,998849,995438,993412,990772,997913\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 551\n", + "Returned IDs: 991335,998500,997688,999647,998139,998329,990092,994699,997192,992357,996727,997371,995370,992546,990934,990366,999898,994778,996576,992994,990315,990505,999343,995203,990231,992330,996646,999201,994831,990957\n", + "Ground Truth: 991335,998500,990650,997006,995985,997688,993174,999647,995942,994776,998139,998329,990092,994699,997192,992357,996727,997371,995370,992546,990934,990366,999898,995906,990980,992822,993034,994778,990866,996576\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 552\n", + "Returned IDs: 995655,991667,997973,992517,997145,993001,991685,999107,990330,990458,991093,998861,998122,997732,997679,992561,998536,997265,991569,996733,996028,995835,997332,999827,995780,995536,990096,997319,995167,997685\n", + "Ground Truth: 995655,991667,997973,992517,997145,993001,991685,999107,990330,990458,991093,995222,993280,998861,990091,998122,997732,997679,992561,998536,997265,991569,996733,996028,998710,992862,995835,997332,999827,993006\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 553\n", + "Returned IDs: 997698,993715,998716,998030,990392,993917,991663,992664,996183,995705,992636,991685,995837,997278,996986,990364,999052,998623,991620,998903,990998,997944,992407,992026,993366,991920,998440,994510,995735,991451\n", + "Ground Truth: 997698,993715,998716,998030,990392,993917,991663,992664,996183,995705,992636,991685,995837,997278,996986,990364,999052,998623,991620,998903,998174,992547,990998,997097,997944,992407,992026,993366,991920,998440\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 554\n", + "Returned IDs: 993305,992350,992258,991152,996163,996669,995248,997540,996620,997395,998892,995542,991705,997689,999579,994301,995772,995986,990019,999685,993255,997538,999298,997547,992150,990569,997169,994072,991977,990093\n", + "Ground Truth: 993305,992350,992258,991152,996163,996669,995248,997540,996620,997395,998892,995542,991705,997689,999579,994301,995772,995986,990019,999685,993255,997538,999298,997547,992150,990569,997169,994072,997523,991977\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 556\n", + "Returned IDs: 990399,993834,999041,993055,994237,998697,995184,999087,995701,996075,996230,999639,998758,992225,991042,991530,995672,998720,993727,995418,999724,995203,994167,994809,991256,994280,991485,995516,993283,998514\n", + "Ground Truth: 990399,993834,999041,993055,995855,994237,998697,995184,999087,995701,992774,991300,996075,996230,999639,998758,992225,999651,991042,993902,991530,995672,998720,993727,995418,991557,997531,999724,995203,994167\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 557\n", + "Returned IDs: 995336,998090,992384,996296,990949,994100,996657,996001,997918,997320,998951,990343,995873,997571,993143,993809,997554,997913,990772,992330,990339,996592,992880,992052,993492,999144,992692,990897,990910,993289\n", + "Ground Truth: 995336,999900,992277,998090,992384,996296,990949,994100,996657,996001,997918,997320,998951,990343,995873,997571,993143,995374,993809,997554,997913,990772,992330,993512,990339,996592,992880,992052,993492,999144\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 558\n", + "Returned IDs: 998934,998720,991459,994167,994280,991130,996001,998593,992429,999639,999173,999216,993834,992437,992441,993993,997337,995158,995701,993375,996328,997790,992225,996526,993848,998864,997847,995895,994610,990096\n", + "Ground Truth: 998934,998720,991657,991459,994167,994280,991130,996001,992429,998593,999639,999173,999216,993834,999408,995103,996408,992437,992441,993993,997337,995158,995701,993375,996328,997790,992225,996526,997166,993848\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 559\n", + "Returned IDs: 996809,998495,990340,995895,997442,994529,990435,999500,993391,999377,994783,998913,997182,994830,999430,994269,999352,992827,990126,994402,996411,995633,991382,995535,992551,991128,996819,992566,993219,995476\n", + "Ground Truth: 996809,998495,990340,995895,997442,994529,990435,999500,993391,999377,994783,998913,997182,994830,999430,996049,994269,999352,992827,990126,994402,996411,995633,991382,995535,992551,991128,996819,992566,993219\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 560\n", + "Returned IDs: 997247,997977,995203,996868,999111,992082,993504,995740,992241,993157,996075,992747,995372,993174,999735,994167,995855,991425,993609,994672,996196,998303,992825,999526,993489,997368,996217,990196,998715,992571\n", + "Ground Truth: 997247,997977,995203,996868,999111,992082,993504,995740,992241,993157,996075,992747,995372,993174,999735,994167,995855,991425,993609,994672,996196,998303,992825,999526,993489,997368,996217,990196,998715,994694\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 561\n", + "Returned IDs: 991753,995066,995401,996607,993003,991680,994106,996813,992448,997975,996304,990319,999054,993597,991639,995864,998910,997544,993642,997091,996695,999759,997176,999646,997075,991795,995490,992653,993209,991592\n", + "Ground Truth: 991753,995066,995401,996607,993003,991680,994106,996813,992448,997975,996304,990319,999054,993597,991639,995864,998910,997544,993642,997091,996695,999759,999646,997176,997075,991795,995702,995490,992653,993209\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 562\n", + "Returned IDs: 996043,997723,995401,991630,998267,990033,991954,997380,990525,991403,999595,993275,991074,990325,995940,991666,998166,994178,999277,993412,999587,995200,994409,999977,991794,994474,999228,997918,996903,998045\n", + "Ground Truth: 997957,996043,997723,995401,991630,998267,990033,991954,997380,991603,990525,991403,999595,993275,991074,990325,995940,992582,991666,998166,994178,999277,993412,999587,995200,996181,994409,999977,991794,997630\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 563\n", + "Returned IDs: 990508,998359,999717,995206,991323,991650,993603,997581,991348,996178,992001,996075,999258,993686,996626,994233,993809,999806,999473,996458,990022,992557,990866,997367,999480,994331,995596,995108,999230,990456\n", + "Ground Truth: 990508,998359,999717,990031,995206,991323,991650,993603,997581,991348,996178,992001,999132,996075,999258,993686,996803,996626,994233,993809,999806,999473,994341,996458,990022,992557,990866,997442,990912,997367\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 564\n", + "Returned IDs: \n", + "Ground Truth: 993591,997036,993015,993553,994275,997170,991989,996906,990368,994021,995923,995345,992585,998039,994663,990955,992798,994386,993561,993164,994379,992014,990845,993725,997357,993813,997920,991168,993697,997933\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 565\n", + "Returned IDs: 992286,993030,995854,997634,997020,994301,998386,995723,996273,998080,993263,998650,990591,992531,994677,997723,991423,996139,997395,993717,990189,991404,995062,993223,995625,993127,990178,999961,999020,992558\n", + "Ground Truth: 992286,993030,995854,997634,997020,994301,998386,995723,997114,996273,998080,993263,992683,998650,995282,990591,992531,994677,997723,994360,991423,996139,997395,990103,993717,990189,991404,995062,993223,995625\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 566\n", + "Returned IDs: 991384,998049,996019,997675,998536,996501,995521,998933,995080,996506,999448,996245,996505,996918,996062,996313,995589,990479,999091,990537,997395,999079,993931,995536,998874,992211,994955,994280,995655,995408\n", + "Ground Truth: 991384,998049,996019,997675,998536,996501,995521,998933,991904,995474,995080,996506,999448,996245,996505,996918,991052,993251,998331,996062,996313,995589,990479,999091,990537,997395,999079,993931,995536,991758\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 567\n", + "Returned IDs: 990974,994541,993637,999009,996996,992551,999898,993807,997320,990096,997004,996216,993695,998633,996830,991179,992927,995895,990786,997410,996564,996944,998302,996809,997126,994782,995638,998950,993053,991154\n", + "Ground Truth: 990974,994541,994557,993637,999009,996996,992551,999898,994638,992553,993807,997320,990096,997004,996216,993695,995906,993928,998633,996830,991179,992927,998006,997577,995895,990786,997410,995755,996564,996944\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 568\n", + "Returned IDs: 993883,992859,990416,991815,992272,996898,994013,993908,990903,997566,999402,996109,998162,997185,990462,995268,993209,999279,998980,995544,991585,999172,997384,997710,998898,991507,999672,997031,998060,994991\n", + "Ground Truth: 993883,992859,990416,991815,992272,996898,994013,993908,990903,997566,999402,996109,998162,997185,998609,990462,995268,993209,999279,998980,995544,991585,999172,997384,993166,997710,998898,991507,995633,999672\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 569\n", + "Returned IDs: 994145,993561,993993,993939,996328,992862,996001,990887,994853,998710,992450,997878,996219,996132,997529,994742,990096,994682,996408,991759,999444,998382,993342,996042,995895,994830,990918,998221,996884,998495\n", + "Ground Truth: 994145,993561,993993,993939,996328,992862,996001,990887,994853,998710,995326,995535,992450,997878,996219,996132,997529,994742,999472,990133,990096,994682,996408,991759,999444,998382,993342,996042,995895,994830\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 570\n", + "Returned IDs: 992293,990662,990097,996001,996657,998549,999853,992521,996028,996147,995283,998878,997305,993848,996984,997554,995401,996368,999099,995873,993053,992862,998593,998017,992395,995158,990811,990211,993725,990573\n", + "Ground Truth: 992293,990662,990097,996001,996657,998549,999853,992473,992521,996028,996147,995283,998878,997305,993433,993848,996984,997554,995401,996368,999099,995873,993053,997466,992862,995002,994531,999587,998593,998017\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 571\n", + "Returned IDs: 993997,998962,990663,999279,997031,996786,997844,990758,996131,995633,996867,996302,993920,993767,995640,995268,990157,994120,993209,997566,995766,995289,996346,999415,996254,996109,994409,999672,994730,995240\n", + "Ground Truth: 993997,998962,990663,999279,997031,996786,997844,990758,996131,995633,996867,998609,996302,993920,993767,995640,995268,990157,994120,999996,993412,993209,993488,997566,995766,995289,996346,999415,996254,996109\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 572\n", + "Returned IDs: 996001,995823,998593,991789,997768,996028,990930,992862,997586,994307,995536,997790,995002,993053,996368,995289,990811,998495,996147,996062,993342,999444,999413,993906,998874,990096,990399,998039,990382,993168\n", + "Ground Truth: 996001,995823,998593,991789,997768,996028,990930,992862,997586,994307,995536,997790,997920,995002,993053,996368,995289,990811,998495,995097,996147,991747,996062,993342,999444,999413,997691,993906,998874,995096\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 573\n", + "Returned IDs: 998171,998866,995923,993390,993561,991996,997822,995547,992660,998045,996042,995052,995701,991958,994409,997289,990137,997058,996867,995289,994396,998165,991011,993433,994140,999868,992576,996125,991415,997191\n", + "Ground Truth: 998171,998866,995923,993390,993561,991996,997822,995547,992660,998045,996042,995052,995701,991958,998323,994409,997289,990137,997058,996867,995289,990706,994396,998165,991011,993433,994140,999868,992576,996125\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 574\n", + "Returned IDs: 991348,995206,996626,992318,993603,993053,995438,990083,998950,996024,991763,993621,994233,992944,991142,993166,993747,996178,994742,993154,995107,996001,993696,999426,995669,990883,996653,996210,991997,990508\n", + "Ground Truth: 991348,995206,995108,996626,992318,993603,993053,995438,990083,998950,996024,991763,993621,994233,992944,991142,993166,993747,996178,994742,993154,992617,998567,990031,995107,998180,996001,993696,997448,999426\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 575\n", + "Returned IDs: 999136,998570,998586,991844,999306,995695,995257,996001,995142,994643,998593,993948,990991,994185,991987,994187,995455,999117,995701,991038,992945,995755,996623,996147,993628,995200,993342,992760,997369,993289\n", + "Ground Truth: 990450,998383,999136,998570,998586,994105,991844,999306,996446,995695,995257,996001,995142,994643,998593,996043,993948,990991,994185,991987,999653,994187,997766,995455,997191,998493,999117,995701,999872,991038\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 576\n", + "Returned IDs: 998623,990998,998536,995408,992052,994743,990096,999287,990255,996809,995203,997618,995536,994682,992558,994742,994510,991629,991515,994955,993910,999724,996483,994954,991977,992636,993948,996930,995237,994906\n", + "Ground Truth: 998623,990998,998536,995408,992052,994743,990096,999287,990255,996809,995203,997618,995536,994682,992558,991630,999107,995864,994742,995401,994510,991629,991515,992933,994955,993910,999724,996483,995198,994954\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 577\n", + "Returned IDs: 997918,996903,995373,995453,994088,991887,995444,991844,995080,994334,992604,994092,991698,995807,994804,997675,998318,995685,991110,997515,997637,992933,999144,997110,999478,994819,996867,993754,994170,996658\n", + "Ground Truth: 997918,993223,996903,995373,993512,995453,994088,991887,995444,996282,991844,995080,994334,992604,994092,991698,995807,993412,994804,997675,998318,995685,991110,997515,997637,992933,999144,997110,999478,994819\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 578\n", + "Returned IDs: 997110,991987,995268,999478,994948,990991,997723,994643,993122,998078,991630,999994,990361,990930,994125,997191,996968,991052,999117,996867,999530,995963,994253,999555,992604,993232,992124,992861,995373,997406\n", + "Ground Truth: 997110,991987,995268,999478,994948,990991,997723,994643,993122,995848,998078,990805,991630,999994,990361,993381,990930,994125,997191,996968,991052,999117,996867,991954,999530,995963,994253,995701,992556,999555\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 579\n", + "Returned IDs: 990195,991585,995216,999302,998060,996147,996001,998549,990975,991958,995873,998200,996219,994925,994013,997554,990887,995701,996657,993813,993561,994076,995079,999343,995555,997377,992384,991125,995016,999402\n", + "Ground Truth: 990195,991585,995216,999302,998060,996147,996001,998549,993377,990975,991958,995873,998200,996219,994925,994013,997920,997554,990887,995701,996657,993813,993561,994076,995079,999343,995555,997377,992384,991125\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 580\n", + "Returned IDs: \n", + "Ground Truth: 997126,997415,990893,994467,994903,993501,993460,997084,991303,998588,997340,990998,993084,993931,992056,991383,999430,999323,997196,990173,990579,994510,998716,991672,996706,999139,995895,997565,996184,994098\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 581\n", + "Returned IDs: 995899,991154,998697,992167,998720,995536,993139,994167,995562,996075,999806,999724,997305,999639,996001,992225,994577,997586,992423,993560,991157,996894,993055,995203,998514,998568,994636,990051,991332,995516\n", + "Ground Truth: 995899,991154,998697,992167,998720,995536,993139,994167,995562,996075,999806,999724,996811,997305,999639,996001,992225,994577,997586,992423,993560,991157,995963,996894,999107,993055,998105,995203,998514,998568\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 582\n", + "Returned IDs: 994806,992598,999852,998256,999192,998903,990069,994175,996633,998716,994830,996960,997202,990806,996495,996537,996301,993335,999628,999126,990998,996328,999372,996153,998761,996011,994608,993502,990985,991769\n", + "Ground Truth: 994806,992598,999852,998256,999192,998903,990069,994175,996633,998716,994830,996960,999441,997202,990806,996495,996537,996301,993335,999628,999126,990998,996328,999372,996153,998761,996011,994608,993502,990985\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 583\n", + "Returned IDs: 992680,995756,996645,993725,991669,999740,999356,991075,992090,993444,999302,990743,995435,991464,996411,993645,990528,992024,990701,999280,996627,993205,996969,998940,993053,996384,994783,995126,992679,995906\n", + "Ground Truth: 992680,995756,996645,993725,991669,999740,999356,991075,997798,992090,993444,999302,990743,999922,995435,991464,996411,991624,993645,990528,992024,999261,990701,999280,992783,996627,993205,996969,998940,993053\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 584\n", + "Returned IDs: 998182,994799,990328,998105,993857,999460,991075,990980,995166,998427,998197,996689,995295,996028,997961,995823,990145,999843,992480,999230,993475,994119,998982,996494,990536,990097,990811,993053,993221,997822\n", + "Ground Truth: 998182,994799,990328,998105,993857,999460,991075,990980,998017,995166,998427,998197,996689,995295,996028,997961,995823,990145,999843,993055,992480,999230,993475,992374,994119,998982,996494,990536,990854,990097\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 585\n", + "Returned IDs: 995755,997301,993484,994742,990096,998246,995942,990958,991743,996001,997614,997371,995701,993853,994682,992862,993048,997320,998294,998921,990949,990515,996805,999801,993342,991498,996521,990146,995132,997913\n", + "Ground Truth: 995755,997301,999533,993484,994742,990096,998246,995942,990958,991743,996001,997614,997371,995701,993853,994682,993412,992862,993048,997320,998294,993948,998921,990949,990515,998002,996805,999801,997533,990545\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 586\n", + "Returned IDs: 992183,991479,990109,995802,998011,994778,993853,998550,994699,995396,990231,990647,994450,993662,999243,993403,997913,999710,995793,991502,998899,998828,998951,991249,995336,992147,990146,995824,990910,992442\n", + "Ground Truth: 992183,991479,990109,995802,998011,992946,994778,999561,993853,998550,994699,995396,990231,997439,991207,990647,994450,993662,999243,993403,995465,996912,997913,999647,996735,991147,990478,999710,995793,997194\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 587\n", + "Returned IDs: 993562,990535,994685,997015,995947,991008,998992,991068,993083,997306,993484,992681,990936,991444,990925,999644,994234,995129,999531,992274,997543,993050,991396,996343,996504,994997,990146,990585,996959,998797\n", + "Ground Truth: 991433,993562,990535,994685,997015,995947,991008,998992,991068,993083,997306,993484,992681,990936,991444,990925,999644,994234,995129,990539,999531,992274,997543,993050,991396,999012,996343,996504,994997,990146\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 588\n", + "Returned IDs: 990845,994925,996906,998593,992703,997453,998039,993561,991958,996062,996001,999729,991789,993906,991989,992334,996237,990899,991011,990096,990076,995289,996311,996277,994424,996834,993064,991869,999491,992576\n", + "Ground Truth: 990845,994925,996906,998593,992703,997453,998039,993561,991958,995162,996062,996001,999729,995401,991789,993906,991989,992334,996237,997028,990899,991011,990096,990076,995289,996311,996277,999340,998207,995560\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 589\n", + "Returned IDs: 990271,997878,990179,991776,995868,994448,992193,996219,994145,990369,994689,994373,990272,997349,996593,995096,991075,990077,999102,990315,995438,995690,994682,994742,993725,991283,999331,999144,992337,995220\n", + "Ground Truth: 990271,997878,990179,995222,991776,995868,990621,994448,992193,996219,998269,995719,994145,990369,996580,994689,994373,990272,997349,996593,995096,991075,998204,990077,999102,990315,995438,995690,994682,994742\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 590\n", + "Returned IDs: 998792,990690,992169,990226,996028,994964,997822,998039,993717,995526,993906,999898,992944,992408,993197,996494,993807,994925,995200,993561,990845,995861,997320,996328,993637,993869,998593,990210,994340,999613\n", + "Ground Truth: 998792,990690,995360,992169,990226,996028,994964,997822,998039,993717,995526,993906,997732,999898,992944,992408,993197,990580,996494,993807,994925,995200,993561,990845,995906,995861,997320,996328,991780,993637\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 591\n", + "Returned IDs: 993433,996042,995289,996607,992448,998910,995146,993561,995820,995018,997023,995200,996769,995521,998162,995864,996112,996815,991671,999611,992576,991699,996867,991958,994409,997822,995766,999054,997406,998564\n", + "Ground Truth: 993433,996042,995289,996607,992448,998910,995401,995146,993561,996809,995820,995018,997023,995200,996769,995521,998162,995864,996112,996815,991671,999611,992576,991699,996867,997723,991958,994409,997822,995766\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 592\n", + "Returned IDs: 994315,997059,990853,993991,995808,996763,995159,994354,990956,995184,992526,996189,997949,995873,992052,994175,992427,991111,994621,994377,996277,991981,997041,990611,992981,994488,995047,991769,994185,992285\n", + "Ground Truth: 994315,997059,990853,993991,995808,996763,995159,994354,990956,995184,992526,996189,991864,997949,995873,992052,994175,992427,991111,997233,994621,994377,996277,991981,997041,990611,992981,994488,995047,991950\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 593\n", + "Returned IDs: 992558,993255,994280,992790,995401,997395,995886,995134,990214,991404,999715,996762,993478,991977,997918,990538,992685,999305,995119,995154,995968,990969,992933,993143,999179,999154,991382,998950,999961,997822\n", + "Ground Truth: 992558,993255,994280,992790,995401,990790,997395,995374,995886,995134,990214,991404,999715,995145,996762,993478,991977,997918,990538,992685,999305,995119,996721,995154,995968,991052,998045,990969,992933,999461\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 594\n", + "Returned IDs: 998314,990096,999282,990583,993086,999079,996733,999194,996209,999384,993744,993965,997821,999287,991243,992407,996893,993595,990072,999831,990920,998207,998536,995736,999411,997442,998510,991093,996124,999320\n", + "Ground Truth: 998314,990096,999282,990583,993086,999079,996733,999194,996209,999384,993744,993965,997821,999287,991243,992407,996893,993595,990072,999831,990920,998207,998419,998536,995736,999411,992112,997442,998510,993609\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 595\n", + "Returned IDs: 991828,992186,990580,991840,990310,997539,994922,999076,996402,996996,993036,993577,994005,990138,995515,996449,996478,991609,991172,999846,996840,993143,998455,995156,998711,990615,990957,993598,992611,995525\n", + "Ground Truth: 991828,992186,990580,991840,990310,997539,994922,999076,996402,996996,993036,993577,994005,990138,995515,996449,996478,994700,991609,991172,999846,993278,996840,993143,992207,991519,998455,995156,995767,998711\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 596\n", + "Returned IDs: 995981,997620,999374,997304,997437,998246,995596,992357,994266,999331,997913,998697,992748,994100,995132,990588,997429,993186,997320,998974,994894,995638,990949,997614,996075,992384,992124,998353,991045,990866\n", + "Ground Truth: 995981,997620,999374,997304,997437,998246,995792,995596,992357,994266,999331,997913,998697,992748,992047,999710,994100,995132,990588,997429,993412,993186,994916,992583,997320,998974,994894,995638,990949,997614\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 597\n", + "Returned IDs: 995144,990116,996878,990325,997420,991145,999144,993727,995130,995357,993805,998045,997766,996811,995289,995562,996877,994051,996918,999365,993121,993433,997822,992558,993275,992718,990956,991131,992052,998758\n", + "Ground Truth: 995144,990116,996878,990325,997420,991145,999144,997723,993727,995130,993504,995357,993137,992315,993805,998045,997766,996811,995289,995562,996877,999527,995374,994051,996918,990527,999365,993121,993433,992933\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 598\n", + "Returned IDs: 992636,996986,995467,997340,999334,990438,990257,993715,993917,990951,999724,992282,991144,998671,996594,996183,993139,993055,997546,999373,997168,993342,993448,995326,990353,996408,995746,997577,998348,993224\n", + "Ground Truth: 992636,996986,995467,997340,999334,994680,990438,990257,993715,993719,993917,990951,999724,992282,991144,998671,996594,996183,993139,993055,997546,999373,997168,993342,993448,995326,990353,996408,995746,997577\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 599\n", + "Returned IDs: 996298,991464,999922,992240,992408,993725,990145,994472,996627,990531,990091,995295,993444,993906,997453,994765,990226,991021,993053,995126,998055,999126,999139,997944,992322,994663,992703,992169,992910,995680\n", + "Ground Truth: 996298,991464,999922,992240,991663,992408,993725,990145,994472,996627,990531,990091,995295,993444,993906,997453,994765,990226,991021,993053,995126,998055,999126,999139,997944,992322,994663,992703,991168,990712\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 600\n", + "Returned IDs: 996610,997787,991603,998906,996047,996388,990033,993390,995289,991463,997406,990681,998045,990940,994161,998165,993561,996112,992125,991367,991954,996769,998999,995182,992160,997046,995497,997832,995318,995052\n", + "Ground Truth: 996610,997787,991603,998906,996047,996388,990033,993390,995289,991463,997406,990681,998045,990940,994161,998165,995040,993561,996112,992125,991367,991954,996769,998999,995182,992160,997046,995497,997832,995318\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 601\n", + "Returned IDs: 995819,994119,995401,991943,994148,992832,994817,990814,991151,993857,996760,996328,999099,991747,999711,992595,992582,992289,996064,995295,990127,998067,990531,991569,994280,998536,997319,999107,995476,996689\n", + "Ground Truth: 995819,994119,995401,991079,991943,994148,992832,994817,992318,990814,991151,993857,996760,996328,999099,990769,991747,999711,992595,992582,992289,996064,997452,995295,990127,990531,998067,991569,995536,994280\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 603\n", + "Returned IDs: 997878,991464,993561,996834,998705,994838,998382,990315,993725,993166,993053,991958,994730,995679,994145,995435,995283,999151,999173,995384,994338,990621,990441,999740,990614,999302,992261,997529,996411,999126\n", + "Ground Truth: 991024,997878,991464,993561,996834,998705,994838,998382,996752,990315,993725,999472,996565,993166,993053,991958,994730,995679,994145,995435,995283,999151,999173,996943,995384,999352,994338,990621,990441,999740\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 604\n", + "Returned IDs: 994297,995244,994964,993452,994919,998500,999821,998837,999444,992169,995135,998909,991991,999545,999740,994922,991999,990736,990432,997984,996351,991840,998553,995336,998523,997192,999676,997320,994318,998921\n", + "Ground Truth: 994297,995244,994964,993452,991438,994919,998500,999261,999821,998837,999444,990064,992169,995135,996854,995591,998909,991991,999545,999740,994922,991999,990736,990432,997984,996351,991840,998553,995336,991751\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 605\n", + "Returned IDs: 999375,997152,991960,998196,994193,991758,994119,999788,998274,992056,991515,998543,994742,994541,997452,990814,991334,999981,993180,997967,992077,998913,990796,996302,999543,990722,998855,994893,993744,995200\n", + "Ground Truth: 999375,997152,992468,991960,998196,994193,991758,994119,999788,998274,992056,991447,998466,995438,991515,998543,994742,994541,995494,997452,990814,991334,999981,990127,993180,997967,992077,998913,990796,996302\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 606\n", + "Returned IDs: 992195,993708,992907,994922,997135,995588,995693,995365,999784,991157,993696,999676,992039,991015,992612,998837,997379,995203,990310,998123,993143,996840,993355,991485,999077,994964,996727,998950,991848,997305\n", + "Ground Truth: 992195,993708,992907,994922,997135,995588,995693,995365,999784,991157,993696,999676,992039,991015,992612,998837,997379,995203,990310,998123,993143,996840,993355,991485,999077,997037,994964,993633,996727,998950\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 607\n", + "Returned IDs: 990146,999600,990958,990863,993083,993562,994964,994685,992301,994217,994425,996959,996532,990925,992968,995947,995884,993238,997320,999262,998302,994673,995716,999374,994922,991743,993168,995372,991008,998170\n", + "Ground Truth: 990146,999600,990958,991086,990863,993083,993562,994964,994685,992301,994217,995909,994425,996959,996532,999444,990652,990925,992968,997876,995947,995884,993238,997320,999262,998302,993412,994673,995716,999374\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 608\n", + "Returned IDs: 991569,997009,992561,996328,990437,993931,990040,997925,997611,990293,993001,991667,995159,997513,998720,990330,997973,990956,994509,998593,997204,995655,990364,995701,992832,990951,996408,991679,998067,998156\n", + "Ground Truth: 991569,997009,992561,996328,990437,993931,990040,997925,997611,990293,993001,993778,996706,991667,995159,997513,998720,990330,994796,997973,990956,994509,998593,997204,995655,990364,995701,992832,992850,990133\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 609\n", + "Returned IDs: 993053,998153,993721,994751,992680,993444,999740,998435,995861,998495,996685,992679,994558,991382,991128,995735,998055,998454,993984,999356,999864,993725,999280,998761,997933,996645,992079,998221,991075,994742\n", + "Ground Truth: 993053,998153,993721,994751,992680,993444,999740,998435,995861,998495,996685,992679,994558,991382,991128,995735,999295,998055,998454,993984,999356,999864,993725,999280,998761,997933,996645,992079,998221,993123\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 610\n", + "Returned IDs: 991638,997732,994956,993929,993765,992655,998107,992120,995861,992813,999827,992169,998458,992408,990226,997852,991334,994131,993877,990889,990961,997453,996328,994557,998495,997822,993080,997969,990382,998171\n", + "Ground Truth: 991638,997732,994956,993929,997112,993765,992655,998107,992120,995861,992813,999985,999827,992169,998458,992408,990226,997852,990441,991334,994131,993877,990889,990961,997453,990266,996328,994557,998495,997822\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 611\n", + "Returned IDs: 999056,999426,996744,996500,992167,996075,991984,992041,992927,993766,991823,993283,997661,995596,996761,995132,993237,999041,996169,996001,995516,994809,993834,995016,991392,998232,995855,996657,996576,992351\n", + "Ground Truth: 999056,999426,996744,996500,992167,996075,991984,997777,992041,992927,993766,995291,991823,993283,993156,997661,995596,996761,995132,993237,995375,999041,999132,996169,996001,995516,995898,994809,993834,998990\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 612\n", + "Returned IDs: 994830,994742,998950,993588,996809,992807,998913,998823,994806,995552,994269,992318,996805,991154,999627,990392,993342,998495,999430,992598,992551,992427,993053,995899,995002,994682,992415,997073,995289,999377\n", + "Ground Truth: 994830,994742,998950,993588,996809,992807,998913,998823,994806,995552,994269,992318,996805,991154,996301,999627,990392,991390,993342,998495,995299,999430,992598,992551,994751,992427,993053,995899,995401,995002\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 613\n", + "Returned IDs: 994051,990256,991860,995516,992915,991799,995755,990862,992596,992165,997437,993396,999725,997305,996811,999251,991052,994879,995963,992546,991690,995184,996406,990958,996942,991982,996918,990096,994266,997154\n", + "Ground Truth: 994051,990256,991860,995516,997645,992915,997811,991799,995755,990862,992596,992165,997437,995132,997429,993396,999725,993336,992238,997305,996856,996811,999251,994335,991052,994879,992357,995963,990178,992546\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 614\n", + "Returned IDs: 995769,995850,999606,998837,997275,993237,995004,990043,996686,994612,991407,990514,997486,994785,992612,995557,998657,991307,994967,995515,990580,991510,999714,993091,993570,998244,990965,995788,996402,992907\n", + "Ground Truth: 995769,998035,995850,999606,998837,997275,993237,995004,990043,996686,994612,991407,999888,998101,990514,997486,994785,992612,995557,998657,991307,994967,996561,995515,990580,991510,999714,993091,993570,998244\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 615\n", + "Returned IDs: 996116,990701,997374,997207,998913,997209,992024,994120,990126,995704,994168,995482,997628,992599,993240,993384,990614,994730,999402,992156,993857,994728,995633,998982,991943,996481,990012,998495,998156,998722\n", + "Ground Truth: 996116,990701,997374,994350,997207,998913,997209,992024,994120,990126,995704,999586,991156,994168,995482,997628,992599,992251,992871,993240,993384,990614,994730,999402,992156,993857,994728,995633,998982,991943\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 616\n", + "Returned IDs: 998566,997193,995691,997126,990305,993566,996194,990184,991636,996421,993127,998466,995482,990189,991157,990776,993678,991938,995883,992944,990948,991783,993920,999158,993181,996222,997674,998104,991052,994835\n", + "Ground Truth: 998566,997193,995691,997126,990305,994064,993566,996194,990184,996451,991636,996421,993127,998466,995482,990189,991157,996170,990776,993678,999025,999586,992704,991938,992841,995883,990827,990052,995620,992944\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 617\n", + "Returned IDs: 991128,999282,990096,992557,999045,999079,999831,994661,999411,993721,997442,998510,997766,993744,996209,998302,997821,993139,992209,997906,999806,996494,999384,996893,990584,995206,991650,993215,994416,998359\n", + "Ground Truth: 991128,999282,990096,992557,997418,999045,999079,999831,994661,999411,993721,997442,998510,997524,997766,993744,996209,998302,997821,993139,992209,997906,998419,999806,996494,999384,996893,990584,995206,991650\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 618\n", + "Returned IDs: \n", + "Ground Truth: 998888,999555,991944,990991,999584,991100,992638,991188,997154,997850,996393,995655,990167,998006,993948,993749,990423,990558,990154,994727,994467,993381,997030,993956,990361,994374,991836,994643,991385,991596\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 619\n", + "Returned IDs: 997995,997455,995095,996609,997993,998221,990432,993219,995401,992619,991836,990435,996088,999820,990126,994587,995354,999923,992979,997185,995289,993412,999160,993656,995633,995374,999240,998297,994993,999021\n", + "Ground Truth: 997995,997455,995095,996609,992281,997993,998221,992087,990432,993219,991445,995401,992619,991836,990435,996088,993209,999820,990126,994587,995354,999923,992979,997185,995289,993412,999160,990157,993656,995633\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 620\n", + "Returned IDs: 997652,997897,998428,992052,993621,998522,993171,990956,999002,999179,998250,995421,992028,992747,990409,990626,994280,994765,995213,994742,991605,996328,996461,999927,990364,993175,991212,996001,992437,993347\n", + "Ground Truth: 997652,997897,998428,992052,993621,998522,993171,990956,999002,999179,998250,995421,992028,992747,990409,997368,990626,994280,994765,995213,994743,997723,994742,997494,991605,996328,996461,999587,999927,990364\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 621\n", + "Returned IDs: 995941,997371,990645,993507,994757,993818,998246,991782,995367,995755,996890,997192,994281,990994,997194,992053,990863,999374,991379,993518,997984,995353,999290,992978,991498,993852,995090,990094,990917,990647\n", + "Ground Truth: 995941,997371,990645,993507,994757,993818,998246,991782,995367,995755,996890,997192,993005,994281,990994,997194,994183,992053,990863,999374,991379,993326,991059,993518,997984,995353,999247,997022,990969,993739\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 622\n", + "Returned IDs: 993539,993564,992167,992225,998758,999216,995701,994167,999144,999219,991459,993834,997258,995562,992429,993766,999783,992384,996368,991256,999041,993342,994906,996744,995132,999899,998232,993139,998720,992069\n", + "Ground Truth: 993539,993564,992167,992225,998758,999216,995701,994167,999144,999219,991459,993834,997258,995562,992429,993766,999783,992384,996368,991256,990041,999041,993342,996283,993517,994906,996744,995132,999899,999677\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 623\n", + "Returned IDs: 992558,999564,992738,995724,998827,994372,997124,990273,993077,995983,991009,997158,999146,996918,990538,992170,992179,994163,991260,991905,991157,997775,999639,995401,993706,992615,990479,998142,997859,999908\n", + "Ground Truth: 992558,999564,992738,995724,998827,994372,993140,997124,990273,993077,995983,991009,995281,997158,991585,999146,996918,990538,992170,992179,997411,994163,991260,991905,991157,997775,995411,999639,994742,995401\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 624\n", + "Returned IDs: 992832,998543,992595,997319,992713,990889,995633,991850,996328,998750,994891,992239,997698,999788,992077,991981,994742,999833,990516,992561,990270,992318,999827,998525,995030,990814,991769,999767,991590,999924\n", + "Ground Truth: 992832,998543,992595,997319,992713,990889,995633,991850,996328,998750,994891,992239,997698,991731,999788,992077,991981,994742,999833,990516,992561,990270,995222,992318,999827,998525,995030,990814,991769,999767\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 625\n", + "Returned IDs: 997977,992823,993504,995168,994468,991425,992082,997618,994672,996217,998303,995536,995231,993260,990196,997002,993071,997790,997918,995740,999111,996001,994955,995203,991863,995167,996884,990350,992862,993489\n", + "Ground Truth: 997977,992823,993504,995168,994468,991425,992082,997618,994672,996217,998303,997140,998007,995536,995231,993260,990196,997002,993071,997790,997918,995740,999111,996001,994955,995203,991863,995167,996884,990350\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 626\n", + "Returned IDs: 994906,993289,993342,999587,992664,993820,991459,995401,994946,992636,998855,998536,993275,990399,998623,999977,996764,997002,996001,999820,999724,992179,996147,997910,994743,997790,993715,998831,999453,994955\n", + "Ground Truth: 994906,993289,993342,999587,992664,993820,991459,995401,994946,992636,998855,998536,993275,990399,998623,999977,997332,996764,997002,996001,999820,999724,996371,992179,996147,997910,994743,997790,993715,998831\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 627\n", + "Returned IDs: 992636,990998,996183,996408,998379,993335,994682,996594,991919,992026,999334,996328,993715,990096,990392,994830,993342,990173,997340,991007,993448,992764,994742,995695,995299,998716,990027,990257,993001,992479\n", + "Ground Truth: 992636,990998,996183,996408,998379,993335,994682,996594,991919,992026,991620,999334,996328,993715,990096,990392,994830,992933,995326,993342,990173,997340,991007,993448,992764,994742,995695,995299,998716,990027\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 628\n", + "Returned IDs: 999457,990194,998279,999555,999994,999205,993461,990167,999223,998674,999430,992628,993076,994109,993224,998586,998570,991610,995234,995848,991025,998086,998078,995907,997506,994731,990402,997205,991987,997084\n", + "Ground Truth: 999457,990194,998279,999555,999994,999205,993461,990167,999223,998674,999430,992628,998072,993076,994109,993224,998586,998570,991038,997415,991610,995234,995848,991025,999985,998086,998078,995907,995082,997506\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 629\n", + "Returned IDs: 995135,990898,998751,999649,998660,991172,991840,996854,998011,995365,999676,998915,997291,993640,990965,993253,995693,993404,991999,994415,998540,997192,994497,996976,993143,993809,997403,991833,992087,995090\n", + "Ground Truth: 993250,995135,990898,998751,999649,998660,995865,991582,991172,991840,996854,994819,998011,998996,996352,997641,995365,999676,998915,997291,993640,997479,990965,990310,993253,995693,993404,990979,991999,994415\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 630\n", + "Returned IDs: 994852,992190,999451,997106,990884,999797,994622,996894,995138,995752,992349,993789,997818,993355,999645,997076,990096,996884,993706,998974,992668,996196,999807,999639,996262,990522,995210,993222,990957,995436\n", + "Ground Truth: 994852,992190,999451,997106,990884,999797,994622,996894,995138,998455,995752,992349,993789,997818,993355,999645,997076,990096,996884,995769,993706,998974,992668,996196,999807,991637,999639,992612,996262,990351\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 631\n", + "Returned IDs: 999934,999225,994529,997837,999372,990593,993856,990226,990126,995083,990340,998338,995300,992859,995633,990435,994742,992450,996042,995861,993100,992434,993731,996328,999830,991746,993433,999672,992449,993561\n", + "Ground Truth: 999934,999225,994529,997837,999372,990593,993856,990226,990126,995083,996372,990340,998338,995300,992859,995633,990435,994742,992450,996042,996306,995861,993100,992434,997185,993731,996328,999830,991746,993433\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 632\n", + "Returned IDs: 995210,996808,997918,996062,998436,993512,990366,993289,990891,997002,991425,997332,994821,990885,994042,993260,994959,996001,993504,990622,995826,994694,991833,991311,993157,998303,990491,996551,992571,994200\n", + "Ground Truth: 995210,996808,997918,996062,998436,993512,990366,993289,990891,997002,991425,997332,994821,992711,990885,994468,994042,993260,992612,994959,997531,996001,993504,990622,995826,998611,994694,991833,993268,999091\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 633\n", + "Returned IDs: 999854,998228,993637,998436,997790,999827,996062,990557,995678,994472,997918,990091,997002,998039,999516,996001,993906,995502,997586,992576,991994,992161,995107,993484,997453,992395,990080,994964,998416,993875\n", + "Ground Truth: 999854,990584,998228,993637,998436,997790,999827,996062,990557,995678,994472,997918,990091,997002,998039,993238,999516,996001,993906,993765,995502,997586,992576,997920,991994,992161,995107,993484,997453,992395\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 634\n", + "Returned IDs: 990545,995260,992119,995942,997320,997913,994964,994100,998550,996143,996351,999647,992546,993877,995886,998495,998500,990958,991743,990648,998302,996219,997371,996554,998458,997984,994164,990936,999331,999374\n", + "Ground Truth: 990545,995260,992119,995942,997320,997913,994964,993154,994100,998550,996143,996351,999647,992546,993877,995886,998495,991720,998500,990958,991743,990648,998302,996219,997371,993238,994898,996554,998458,997984\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 635\n", + "Returned IDs: 995702,991973,992521,992703,997822,993293,998162,994409,990930,995766,994140,994742,998779,996368,993738,993561,991185,997723,995648,993831,998633,998165,997320,990845,995289,992448,995216,995052,995679,994711\n", + "Ground Truth: 995702,991973,992521,992703,997822,993293,998162,994409,990930,995766,994140,994742,999134,998779,996368,993738,993561,991185,997723,995648,993831,991603,998633,998165,997320,996042,990845,998999,995289,992448\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 636\n", + "Returned IDs: 995557,996596,990310,993748,998899,993237,995365,991593,993279,996748,995588,993091,998917,995794,993675,992195,994856,998997,998057,999039,994263,992085,995072,994964,997275,992059,996007,999784,993083,991157\n", + "Ground Truth: 995557,999604,996596,990310,993748,999592,998899,993237,995365,991593,993279,996748,995588,993091,992207,998917,995952,995794,993675,992195,994856,998997,998057,999039,998654,994263,992085,995072,994005,991519\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 637\n", + "Returned IDs: 998357,995965,993500,991977,993975,993583,994614,992376,993077,994760,996663,999171,998406,998142,999763,995408,992535,991740,996158,991432,994165,990249,993786,990908,993609,993839,992494,998879,995459,993810\n", + "Ground Truth: 998357,995965,993500,991977,993975,993583,994614,992376,993077,994760,996663,999171,998406,998142,999763,995408,992535,991740,996158,991432,994165,990249,993786,999008,990908,993609,993839,992494,998879,995459\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 638\n", + "Returned IDs: 999638,990926,992349,992718,997816,997859,994163,991540,996894,995138,996318,994010,991498,991977,998697,998417,997113,998118,998334,999387,994964,992124,994165,990908,990343,995536,994972,992558,990083,990772\n", + "Ground Truth: 999638,990926,992349,992718,997816,997859,994163,991540,993948,996894,995138,996318,994010,991498,991977,998697,998417,997113,998118,998334,999387,994964,992124,994165,990908,990343,995536,994972,991637,999908\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 639\n", + "Returned IDs: 994137,999358,995378,990560,990390,999363,991900,998462,995942,998966,992310,991396,993403,990146,991444,997972,993484,990742,994964,994479,999110,999390,997539,999710,990311,996629,998365,995949,996959,994978\n", + "Ground Truth: 994137,999358,995378,990560,990390,999363,991900,998462,995942,998966,992310,991396,993403,990146,993072,991444,997972,993484,991178,999229,990742,994964,997880,994479,999110,993284,999390,997539,999710,990311\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 640\n", + "Returned IDs: 996127,993839,999860,999639,993561,991923,990550,994117,995013,991958,992473,996196,998593,991332,993489,990167,996391,994372,991350,996379,995562,998186,995983,998697,995894,996001,998142,998303,990096,995701\n", + "Ground Truth: 996127,993839,998220,999860,999639,993561,991923,990550,990303,994117,995013,991958,992473,996181,996196,998593,991332,993489,990167,996391,994372,999393,991350,991212,996379,995562,994796,998186,995983,998514\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 641\n", + "Returned IDs: 991810,996809,990127,993817,999788,994682,994742,992827,998063,992372,997349,997319,993744,995633,999852,990223,995895,990173,998536,998495,990986,990722,996408,997401,990392,997529,991620,998525,992832,997630\n", + "Ground Truth: 991810,996809,990127,993817,999788,994682,994742,992827,998063,999183,992372,997349,997319,993744,995633,999852,990223,995222,992318,995895,990173,995401,998536,998495,990986,990722,996408,997401,990392,997529\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 642\n", + "Returned IDs: \n", + "Ground Truth: 999338,998921,997292,997301,994942,995659,996516,997873,992451,997984,998721,998445,990512,997668,997320,995306,999271,990560,990405,995045,995961,994964,997168,997286,999904,995623,992176,999526,995874,999533\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 643\n", + "Returned IDs: 997749,999411,994909,995421,994765,993727,992934,990521,994280,991491,996276,994541,998934,995963,997897,995110,994697,994809,993448,999601,992457,996657,995203,990205,992052,995456,991850,998522,992933,998779\n", + "Ground Truth: 997749,999411,994909,995421,994765,993727,992934,998123,990521,994280,999453,991491,996276,990844,994541,993678,998934,995963,997897,990998,995110,994697,998510,994809,992112,990021,993448,999601,992457,999179\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 644\n", + "Returned IDs: \n", + "Ground Truth: 996296,991456,990176,997022,994842,996075,990742,997699,992931,992880,997178,990957,997918,991131,990949,995417,998540,993253,996592,993696,998967,992442,991634,995336,993529,992942,994100,999710,992277,995981\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 645\n", + "Returned IDs: 997867,991740,993839,992376,998357,991977,993583,998879,995965,990908,993647,997407,994614,999008,993975,990894,991225,995491,997547,996663,992333,995459,993681,996585,999022,994254,993500,994398,999763,999387\n", + "Ground Truth: 997867,991740,993839,992376,998357,991977,993583,998879,995965,990908,993647,997407,994614,991064,999008,993975,990894,991225,995491,997547,996663,992333,995459,993681,996585,999022,994254,993500,994398,999763\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 646\n", + "Returned IDs: 997643,991309,994660,992638,991797,998277,994393,992943,997683,999530,993566,993749,994294,997568,993666,999156,997110,994656,997030,995044,997429,996393,990554,997114,998888,995253,990132,991937,998858,992369\n", + "Ground Truth: 997643,991309,994660,992638,991797,998277,994393,992943,997683,999530,993566,993749,994294,997568,993666,999156,997110,994656,997030,992926,995044,997429,996393,990554,995994,997114,998888,995253,990132,991937\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 647\n", + "Returned IDs: 990679,998734,995250,999473,993154,997370,994789,995203,999898,996075,996623,997918,990352,993484,990652,994829,990470,993646,996452,999876,999384,990957,992548,996001,990123,999653,999017,990448,997688,994268\n", + "Ground Truth: 990679,994552,998734,995855,995250,999473,993154,997370,994789,995203,999954,994212,999898,996075,996623,997918,993243,990352,993484,990652,994829,990470,993646,990809,996452,999876,999807,999384,999132,997920\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 648\n", + "Returned IDs: 994125,990641,996609,990655,992604,997918,995401,991844,995289,998913,991996,996809,995718,996411,996196,996075,993725,995274,990126,998999,994603,993561,998495,995881,995633,996918,995886,990096,995200,995455\n", + "Ground Truth: 994125,990641,996609,990655,992604,997918,995401,998383,997403,991844,998913,995289,991996,996809,993529,993412,994317,995718,993433,996411,996196,991179,996075,993725,995274,990126,997906,998999,994603,993561\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 649\n", + "Returned IDs: 996360,998045,993561,998545,991052,991958,993433,996042,995702,996322,994409,997822,991923,996867,994419,996769,995679,995648,994549,994280,997868,997386,998564,995536,997674,997675,997289,990137,999530,997838\n", + "Ground Truth: 996360,998045,993561,998545,991052,991958,999809,993433,996042,995702,996322,994409,990882,997822,991923,993412,996867,994419,996769,995679,995648,994549,994280,997868,997386,995873,998564,995536,997674,997675\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 650\n", + "Returned IDs: 993839,996127,996658,991432,998372,995340,990061,990487,991028,997864,991923,993976,998370,991743,991350,996391,990638,995691,994514,995515,993583,998779,998879,997914,995453,992929,990602,998853,996742,996379\n", + "Ground Truth: 993839,996127,996658,991432,998372,993718,995340,990061,990487,991028,998079,997864,991923,993976,998370,991743,991350,996391,992813,990638,995691,995515,994514,993583,998779,998879,997914,998686,995453,992929\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 651\n", + "Returned IDs: 995562,992315,990596,998303,991037,999144,997420,991332,993489,996562,992819,990834,997926,996908,996286,992199,993275,999435,998331,995894,998495,996233,993609,995349,998855,990630,997506,991425,995536,993560\n", + "Ground Truth: 995562,992315,990596,998303,991037,999144,997420,991332,993489,997531,996562,992819,991212,990834,997926,996908,996286,995740,992199,993275,999435,998331,995894,998495,996233,994946,993609,995835,995349,998855\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 652\n", + "Returned IDs: 996646,997320,999647,997913,994450,993253,999109,997470,999948,999324,998550,990650,992950,992969,993387,998302,997826,999701,994266,991593,993031,990066,992972,992147,994604,993579,999036,991479,993662,999535\n", + "Ground Truth: 999328,996646,997320,999647,997913,994450,993253,999109,997470,999948,991743,999324,998967,998550,998287,990182,994192,990650,997921,996296,992950,992969,999390,999258,993387,998302,997826,999701,994266,998644\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 653\n", + "Returned IDs: 996690,992608,994384,996196,997775,992215,997902,991772,996140,993670,994163,992661,995983,993706,991550,998950,996726,995019,991009,996262,990538,998697,990824,999639,998364,991822,995552,995972,997368,993500\n", + "Ground Truth: 996690,992608,994384,996196,997775,992215,997902,991772,996140,993670,994163,992661,995983,993706,991550,999908,998950,996726,995019,991009,996262,990538,998697,990824,999639,998364,991822,995552,995972,997368\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 654\n", + "Returned IDs: 997192,993852,997571,998844,994281,999374,996554,994850,999953,995833,990275,992147,992183,994218,990432,992309,998011,998263,995354,999144,993529,995336,999317,993443,999883,999804,992854,996075,992452,996721\n", + "Ground Truth: 995374,993636,997192,993852,997571,998844,994281,999374,996554,994850,999953,995833,990275,992147,992183,994218,990432,992309,998011,998263,995354,999144,993529,995336,999317,993443,999883,999562,999804,992854\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 655\n", + "Returned IDs: 996247,991459,999804,990294,999639,995894,997918,991979,994125,996407,991630,996500,994911,998022,992473,996918,992516,992429,995932,990897,992558,993188,999306,993906,992293,995701,995336,998758,998697,993848\n", + "Ground Truth: 996247,991459,999804,990294,999639,995894,997918,991979,994125,996407,995132,991630,996500,997957,994911,998022,992748,992473,993948,996918,992516,992429,995932,990897,992927,992558,993188,999306,993906,991657\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 656\n", + "Returned IDs: 999544,999676,992348,999453,994800,991787,999452,995968,992612,994195,993683,991673,996309,992907,992309,992739,990080,993781,991485,995372,999686,998607,999254,994964,991999,991510,999210,990334,990233,991427\n", + "Ground Truth: 999544,999676,992348,999453,994800,991787,999452,995968,992612,994195,993683,991673,996309,992907,992309,992739,990080,993781,991485,995372,996352,999686,998607,992298,999254,994964,993633,991999,995365,999951\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 657\n", + "Returned IDs: 995861,994751,990091,998055,990133,993939,992780,999673,997103,999740,991334,993730,995191,999378,993877,992240,990145,997026,998610,995895,990712,990531,992652,996328,994472,991390,995735,999458,990705,995326\n", + "Ground Truth: 995861,994751,990091,998055,990133,993939,992780,999673,997103,999740,991334,993730,995191,999441,996752,999378,993877,992240,997077,997933,990145,997026,999436,998610,995895,996566,990712,990531,993118,992652\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 658\n", + "Returned IDs: 994488,994416,992318,999060,993772,996809,997332,990843,998950,991650,994541,997679,996001,993603,994955,993347,993721,991142,990930,993079,995899,995678,991409,995167,999320,996024,993293,999643,990096,992944\n", + "Ground Truth: 994488,994416,992318,999060,993772,996809,999179,997332,990843,998950,991650,994541,997679,996001,993603,998543,994955,998218,993347,993721,991142,990930,993079,995899,995678,991409,995167,999320,996503,996024\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 660\n", + "Returned IDs: 990516,995948,994893,995633,995030,998557,997319,996747,996867,994009,992289,999235,996672,994283,994742,995190,995289,991810,996302,999957,997401,995558,999875,997406,999160,992239,998196,998750,994204,995302\n", + "Ground Truth: 990516,995948,994893,995633,995030,998557,997319,996747,996867,994009,992289,999235,996672,994283,994742,995190,995289,991810,996302,993856,999957,997401,995558,999875,997406,999805,995348,999160,992239,998196\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 661\n", + "Returned IDs: 993461,990479,996918,991052,996535,990897,998267,993014,999646,997723,997586,990041,991758,998564,998518,999980,995963,997910,994282,992979,997675,997216,991891,992068,991666,992558,990942,998067,990256,997420\n", + "Ground Truth: 993461,990479,996918,991052,996535,990897,998267,993014,999646,997723,997586,990041,991758,998564,992933,998518,999980,995963,997910,998924,994282,992979,997675,997216,991891,998105,993412,992068,991666,992558\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 663\n", + "Returned IDs: 990225,999555,990167,993076,997506,999373,994834,990361,993366,996393,994374,995257,996408,993001,998495,996797,993224,995218,998536,999430,998888,996209,991100,999994,997154,991610,992470,997182,990173,990589\n", + "Ground Truth: 990225,999555,990167,993076,997506,999373,994834,995082,990361,993381,991881,993366,996393,994374,995257,996408,993001,998495,996797,999841,997142,993224,995218,998536,995222,999430,998888,996209,991100,995326\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 664\n", + "Returned IDs: 994978,992228,993579,991137,995841,999109,993845,991615,997984,998477,990585,990512,991479,995372,997024,998445,995668,999036,999024,995641,995623,993387,997018,996653,999358,999982,996959,997320,991450,993066\n", + "Ground Truth: 994978,992228,993579,992789,991137,995841,999109,993845,991615,997984,998477,990585,990512,991479,995372,997024,991090,998445,995668,999036,999024,995641,991468,991209,995623,993387,997018,996653,999358,993853\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 665\n", + "Returned IDs: 997777,990949,991593,997320,999290,993484,997370,992839,993253,999535,990957,997539,995515,997372,995203,999453,996002,994486,999982,995400,992069,995596,994964,998950,994366,999586,997913,994789,995826,991131\n", + "Ground Truth: 997777,990949,991593,997320,999290,993484,997370,992839,993253,999855,999535,990957,997539,995515,997372,995203,999453,990742,990890,996002,994486,995942,999982,995400,999680,994826,992069,993389,999437,995596\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 666\n", + "Returned IDs: 994954,992521,993438,994409,999866,992448,995640,996368,996769,997868,993561,996987,991969,995820,998255,996187,997091,990005,998205,999250,997466,991349,998045,992426,991967,997554,992937,997612,995536,998075\n", + "Ground Truth: 994954,992521,993438,994409,999866,992448,995640,995873,996368,996769,997868,993561,996987,991969,995820,998255,996187,997091,990005,995701,998205,999250,997466,991349,998045,992426,991967,997723,997554,993517\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 667\n", + "Returned IDs: 997652,993324,993347,995332,991255,990646,991961,993180,993079,996591,997752,990776,997309,996370,999305,996777,998272,996455,994765,996210,997914,992052,994119,994092,995766,993621,998416,997698,996633,994435\n", + "Ground Truth: 997652,993324,993347,995332,991255,990646,999190,991961,993180,996810,993079,996591,997752,990776,997309,994300,996370,999305,996777,998272,996455,994765,996210,997914,991990,992052,994119,994092,995766,993621\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 668\n", + "Returned IDs: \n", + "Ground Truth: 999117,993122,994467,995625,995044,991188,994643,997149,997154,992809,995207,999555,990991,999155,996395,992945,993795,999663,995257,993469,993995,993232,998861,996209,996393,992638,998536,995401,998006,999431\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 669\n", + "Returned IDs: 996489,990442,996981,990249,990256,990209,992558,994334,992017,995332,997380,991426,996811,995552,990538,997155,995449,992179,999860,993473,995401,992170,996196,993515,990620,999715,999908,994661,996429,991337\n", + "Ground Truth: 999901,996489,990442,996981,996186,990249,990256,990209,992558,994334,992017,995332,997380,991426,996811,995552,999480,990538,997155,995449,992179,999860,993473,995401,992170,998213,999859,996196,993515,990620\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 670\n", + "Returned IDs: 998289,998104,993324,992558,994732,990264,998950,996140,995289,994697,993079,997752,994514,997630,998697,996194,995963,995783,996210,996370,994817,993018,997723,992052,997713,991981,993280,993948,993478,998160\n", + "Ground Truth: 998289,998104,993324,992558,994732,990264,998950,996140,995289,996810,990769,995928,994697,993079,997752,994514,997630,998697,996852,996194,992988,995963,995783,995281,996210,996370,994146,995359,994817,993018\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 671\n", + "Returned IDs: 991925,991743,998334,992702,993721,994119,990585,993229,991977,999601,995375,996318,998495,997481,999453,998849,992944,999711,995130,996219,996001,993342,990489,997309,993241,996328,991994,994199,992169,998045\n", + "Ground Truth: 991925,996811,991743,998334,992702,993721,994119,990585,996918,993229,991977,999601,995375,996318,994662,998495,997481,999453,998849,994081,992944,997732,995873,999711,999707,995130,990640,996219,997436,996001\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 672\n", + "Returned IDs: 991618,991163,991969,991349,999731,993813,992521,995820,995813,990603,994140,991117,993738,991795,997868,996277,990930,993561,995701,997928,990193,991922,998205,996169,995864,995640,992448,995395,992851,994409\n", + "Ground Truth: 991618,991163,991969,991349,999731,993813,992521,995820,995813,990603,994140,991117,993738,991795,997868,996277,990930,993561,995701,997928,990193,991922,994353,998205,996169,995864,995640,992448,995395,999866\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 673\n", + "Returned IDs: 999703,992640,993314,993895,991161,993580,992728,990479,998463,992598,994672,993433,994280,998045,990942,994844,993609,995815,996867,999373,994920,995196,996328,996861,993313,990069,997506,998331,995835,994838\n", + "Ground Truth: 999703,992640,993314,993895,991161,993580,992728,990479,999520,998463,992598,994672,993433,994280,991731,998045,990942,990882,994844,993315,998812,993609,995815,999441,992617,996867,999052,999373,997192,994920\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 674\n", + "Returned IDs: 996913,994136,996761,996067,997839,997790,990550,996089,998473,993834,998934,993766,990897,998185,993478,994910,992750,999216,999041,996547,993809,999384,994915,991795,993832,991692,991657,994280,998200,995855\n", + "Ground Truth: 996913,994136,996761,996067,997839,996144,997790,990550,996089,998473,999363,993834,998399,998934,993766,990897,998185,993478,994910,992645,992429,992750,996795,990139,999216,999041,996547,993809,999384,996803\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 675\n", + "Returned IDs: 995467,997506,997168,992636,998536,993715,992282,998045,996769,990438,992521,998021,990951,995375,998697,999338,999526,990096,993637,998921,992825,997586,994016,990930,996408,996594,998097,991245,992576,993001\n", + "Ground Truth: 995467,997506,997168,992636,998536,993715,992282,998045,996769,990438,992521,993681,998021,990951,995375,998697,999338,999526,990096,993637,998921,992825,993719,997586,994016,996918,997723,992933,990930,995562\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 676\n", + "Returned IDs: 999290,997370,996840,992944,993484,998734,995886,993143,999367,996949,996438,993052,992558,993080,995203,991840,990334,999426,991348,990697,998455,997839,990580,995596,993504,996397,992169,999453,990142,998244\n", + "Ground Truth: 999290,997370,996840,992944,993484,998734,996762,995886,993143,999367,996949,996438,993052,992558,993080,995203,991840,990334,999426,991348,990697,998455,990351,997839,990580,995596,993504,996397,992169,999453\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 678\n", + "Returned IDs: 997348,999647,994450,991428,996805,997913,993853,998426,991550,990893,999444,997918,993264,990231,994964,990797,997320,996041,998921,998029,991721,997301,993529,997586,994503,996809,994266,990647,992862,999982\n", + "Ground Truth: 997348,999647,996904,991582,994450,991428,996805,997913,993853,998426,993141,991550,990893,997918,999444,993264,990231,994964,993412,990797,997320,997880,996041,996296,998921,998029,997772,998550,991721,999683\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 679\n", + "Returned IDs: 992604,997598,994187,998697,991844,995455,992128,999136,996645,995755,990540,995435,999306,992133,999867,995409,992051,993560,998022,999740,994125,999478,995780,995268,995527,993820,996649,999356,999795,993561\n", + "Ground Truth: 992604,997598,994187,997212,998383,998697,991844,995455,992128,999136,996645,995755,990540,995435,999306,992133,999867,995409,996129,992051,993560,998022,996915,999740,994125,999478,995780,995268,995527,993820\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 680\n", + "Returned IDs: 996232,996219,994894,998649,999144,995274,991029,992337,999343,994757,996504,996001,995502,999026,993168,990949,999647,997479,995200,990975,990934,991096,999740,995569,998106,996721,992446,997192,998572,998495\n", + "Ground Truth: 997920,996232,996219,993412,994894,998649,999144,995274,991029,992337,999343,994757,996052,999261,999311,996504,991445,995881,995401,996001,995502,994819,999026,993168,993529,990949,999647,997479,995200,990975\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 681\n", + "Returned IDs: 997975,999137,999415,998188,993209,991671,993586,995778,994730,996302,991639,999581,991507,996226,992289,999875,997544,999054,996867,996852,995268,996607,996187,999957,998910,992448,994742,990631,995640,996633\n", + "Ground Truth: 997975,999137,999415,998188,993209,991671,993586,995778,994730,996302,991639,997723,999581,991507,996226,992289,999875,997544,993412,999054,996867,996852,995268,996607,996187,990361,999957,998910,999980,995963\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 682\n", + "Returned IDs: 998104,993799,998536,993609,995963,990392,994119,994711,992521,993324,996760,997752,995783,994409,999129,998162,996987,996328,997309,994280,997420,996867,995401,994541,991712,993180,992569,992832,997320,997176\n", + "Ground Truth: 998104,994300,993799,998536,993609,995963,990392,994119,990879,992617,994711,998568,992521,993324,999840,996760,992131,997723,997752,992289,995783,994409,999129,998162,996987,996328,997309,994280,997420,999886\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 683\n", + "Returned IDs: 999466,998993,995027,995890,996960,996611,998335,995626,992864,998279,990712,996537,990806,992151,998935,992726,991098,996082,996232,991243,999880,994280,995214,990705,996328,994098,993219,996599,993224,994351\n", + "Ground Truth: 999466,998993,995027,995890,996960,996611,998335,995626,992864,998279,990712,996537,990806,992151,998935,992726,995167,991098,996082,991232,996232,991243,999880,994280,995214,996328,990705,994098,993219,996599\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 684\n", + "Returned IDs: 994742,997319,999367,995107,995401,996001,996219,995633,990089,995289,990096,998495,996411,992827,994532,992558,995200,999791,995536,990585,999026,997301,998950,991968,994972,994783,991588,994743,999009,992944\n", + "Ground Truth: 994742,997319,999367,995222,999377,995107,995401,996001,996219,995633,990089,995289,995611,993080,999647,990096,998495,996411,992827,994532,992558,990077,993725,995200,999791,991921,990585,995536,999026,997301\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 685\n", + "Returned IDs: 998184,994964,990361,998779,995204,995289,993238,995274,995702,996867,997822,997320,998999,991588,995633,995216,990845,992703,993648,999160,995052,995766,997192,993831,994742,993848,999587,992994,998458,998165\n", + "Ground Truth: 998184,994964,990361,998779,995204,995289,993238,995274,995702,996867,993673,997822,996959,997320,998999,993209,992334,991588,995633,999026,993389,993412,992582,995216,990845,992703,993648,999160,995052,995766\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 686\n", + "Returned IDs: 994776,999982,999180,998302,992848,992852,992384,998246,997320,997661,994415,997053,990275,997913,992119,991545,993853,999343,998660,994266,996075,990949,998580,992994,990958,993484,992896,998458,991264,991335\n", + "Ground Truth: 994776,999982,990020,999180,998302,992848,992852,997620,992384,998246,997320,997661,994415,997053,990275,997913,992119,991545,993853,997895,999343,998660,994266,996075,990949,998580,992994,990958,996143,993154\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 687\n", + "Returned IDs: 997584,992635,994971,993785,992041,997550,992601,995596,992671,990914,996075,997661,993447,998227,998498,993125,992733,994886,995372,996755,999426,992617,996186,996631,992515,995515,990142,993129,996875,999317\n", + "Ground Truth: 997584,992635,994971,993785,992041,997550,992601,995596,992671,990914,996075,997661,993447,998227,998498,993125,991218,992733,994886,999439,995372,996755,999426,992617,996186,996631,992515,992277,995515,990142\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 688\n", + "Returned IDs: 999812,993504,993024,996429,994964,990818,999339,993260,999111,995895,998303,998715,998950,993391,996216,992747,997586,992571,994672,990409,997977,991458,993489,996196,991946,990494,991206,993906,999840,995560\n", + "Ground Truth: 999812,993504,993024,996429,994964,990818,999339,993260,999111,995895,998303,998715,998950,993391,996216,992747,997586,992571,996677,993805,992979,994672,995622,999735,990409,997977,991458,993489,996196,995887\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 689\n", + "Returned IDs: 992493,997634,995163,992677,998316,992150,992148,993223,997824,998650,995854,994620,993263,995723,992700,995060,991067,993530,995091,997750,996139,991190,996273,995062,991100,998080,999961,993160,997723,994576\n", + "Ground Truth: 992493,997634,995163,992677,998316,992150,992148,993223,997824,998650,995854,994620,993263,995723,992700,995060,991067,995963,993530,995091,997750,996139,991190,996273,995062,991100,998080,999961,993160,991052\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 690\n", + "Returned IDs: 998536,999759,994835,999259,993014,995175,996877,994119,990103,999533,995783,991052,994742,999847,994522,999244,997309,994280,998021,995685,995110,991747,996328,995328,993180,997675,993256,999601,995017,996087\n", + "Ground Truth: 992933,998536,999759,994835,992166,999259,993014,995175,996877,994119,990103,999533,995783,991052,994742,999847,994522,999244,995963,997309,999527,997184,991712,994280,998021,991891,995685,997723,990184,995110\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 691\n", + "Returned IDs: 999045,993721,994718,992551,990096,991409,998495,994269,991650,993053,996001,996075,990843,996936,995361,990584,991382,996219,996631,998803,997550,991451,999302,990091,996101,994472,996016,996013,995476,990786\n", + "Ground Truth: 999045,993721,994718,992551,990096,991409,998495,994269,991650,993053,996001,999336,996075,990843,996936,995361,994355,990584,991382,996219,999577,996631,998803,997550,991451,999302,990091,996101,999343,994472\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 692\n", + "Returned IDs: 991977,993839,993583,996158,994398,992718,996585,993077,994080,993681,994119,992558,990894,992104,994334,999079,999771,994254,998357,990273,995459,992211,996733,992052,992824,995503,999565,990908,990776,998061\n", + "Ground Truth: 991977,993839,993583,996158,994398,992718,996585,993077,994080,993681,994119,992095,992558,990894,992104,994334,999079,999771,994254,998357,990273,995459,992211,996733,992052,992824,995503,990279,999565,990908\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 693\n", + "Returned IDs: 991462,992724,997486,992739,992632,993395,997326,998359,999015,992612,997772,996693,995119,991485,999453,992309,997539,992152,998401,990580,998950,995515,990334,999077,991402,994964,997775,994922,993781,990043\n", + "Ground Truth: 997448,991462,992724,997486,992739,992632,993395,997326,998359,999015,992612,997772,996693,995119,991485,999453,992309,993369,996829,997539,992152,998401,995721,990580,998950,995515,990334,999077,995767,991402\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 694\n", + "Returned IDs: 991425,999794,993504,990956,993665,991552,998615,995349,992457,998303,999111,992137,991648,994672,998109,996884,992351,997506,991037,996012,990658,993609,990409,991257,993429,990959,992640,991779,990885,992819\n", + "Ground Truth: 990010,991425,999794,999211,993504,990956,993665,991552,998615,995349,992457,998303,999111,992137,991648,994672,998109,996884,995541,992110,995103,992351,990752,997506,991037,996012,990658,993609,990409,994681\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 696\n", + "Returned IDs: 993003,992267,996256,992917,994409,997466,991357,999250,995401,992448,997909,990295,992763,995242,996815,994508,994865,998910,998278,994106,993537,992521,997299,996968,998756,991197,995268,999530,995864,998361\n", + "Ground Truth: 993003,992267,996256,992917,994409,997466,999250,991357,995401,992448,997909,990295,992763,995242,996815,994508,994865,998910,998278,994106,993537,992521,996223,997299,996968,998756,991197,995268,992260,999530\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 697\n", + "Returned IDs: 993512,995336,997571,993096,999144,999331,998090,990647,992134,993853,992384,994197,992692,994334,993492,991131,993153,995357,998562,993785,996296,992453,994227,994100,999453,992748,992337,999639,997918,991094\n", + "Ground Truth: 993512,995336,997571,993096,999144,999331,998090,992134,990647,993853,992384,994197,990214,994810,992692,995748,994334,993492,991131,993153,995357,995374,994819,993925,991923,996882,990654,998562,993785,998410\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 699\n", + "Returned IDs: 999984,995589,995408,999543,993936,991011,998855,993778,991977,994113,993433,991962,994191,997944,999643,992576,995582,992179,998623,991556,994254,996277,992052,991515,996733,995113,998536,993281,999091,993979\n", + "Ground Truth: 999984,995589,995408,999543,993936,991011,998855,993778,991977,999461,994113,993433,991962,994191,997944,999643,992576,995582,992179,998623,991556,995401,994254,996277,992052,991515,996733,995113,998536,993281\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 700\n", + "Returned IDs: 990956,996016,991382,992652,992827,993501,994927,998543,994541,998495,996809,995191,999179,994765,995895,993460,998167,996830,997270,991469,996706,999830,993219,996301,993391,997506,992628,999192,997442,994307\n", + "Ground Truth: 990956,996016,991382,992652,992827,993501,994927,998543,994541,998495,996809,994378,995191,999179,994765,995895,993460,998167,996830,997270,995401,991469,996706,999830,993219,996301,998206,993948,990712,993391\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 701\n", + "Returned IDs: 990930,991459,992036,992790,999639,998874,995536,995401,996001,994987,990412,993515,993673,996247,995932,996371,991530,997918,995672,998593,996918,998303,996181,991721,994280,992749,990794,994809,993504,997191\n", + "Ground Truth: 990930,991459,992036,992790,999639,998874,995536,995401,996001,994987,990412,993515,993673,990337,996247,995932,991408,996371,991530,997918,998990,995672,998593,991052,996918,998303,991392,996181,997238,991721\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 702\n", + "Returned IDs: 998039,990146,994925,994964,990925,990845,997969,992334,998593,998302,990560,994419,996001,996028,994853,998921,995923,993831,993561,997453,997822,994307,997586,992944,997320,997046,996785,999374,990529,991685\n", + "Ground Truth: 998039,990146,994925,995162,994964,990925,990845,997969,992334,998593,998302,990560,994419,996001,996028,993648,994853,993412,997920,998921,995923,993831,998228,995906,993561,997453,997822,994307,995401,997586\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 703\n", + "Returned IDs: 998668,994557,991377,991179,990506,995547,995289,998416,999294,993561,991923,996042,995923,997969,997732,994782,998292,997822,995643,993378,996494,996298,998133,996216,996080,997386,998039,993929,998382,990137\n", + "Ground Truth: 998668,994557,991377,991179,999785,990506,995547,995289,995162,998416,999294,993561,991923,996042,995923,997969,991790,997732,994782,998292,997822,995643,993378,993412,996494,996298,994026,998133,996216,996080\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 704\n", + "Returned IDs: 990003,997913,999647,990210,990545,996053,999331,994873,998302,990958,993031,990231,998540,995438,990650,994450,993484,997320,991370,998550,999167,999263,994102,992147,990957,998899,995802,996075,992546,990949\n", + "Ground Truth: 990003,997913,999647,990210,990545,992946,996053,998344,999331,994873,998302,990958,993031,991479,997439,990231,998540,991429,995438,990650,991852,991782,994450,993484,997320,993990,991370,994898,996296,998550\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 705\n", + "Returned IDs: 997897,994280,995421,991605,991447,991232,993489,992052,990627,997442,990844,992864,997926,995626,995203,992457,991332,996817,995630,998428,990885,997790,991218,991212,993175,993391,994738,994531,990630,994351\n", + "Ground Truth: 997897,994280,995421,991605,991447,991232,993489,992052,990627,997442,990844,992864,997926,998071,995626,993087,995203,992457,991332,996817,997531,995630,998428,990885,997790,991218,991212,993175,999550,993391\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 706\n", + "Returned IDs: 998798,995374,996703,990992,997015,991467,992854,996353,998951,990585,994197,991504,997609,990535,999026,996959,994595,997009,991173,991498,999089,996714,992189,996658,993562,995824,995034,998844,999024,995274\n", + "Ground Truth: 993368,998798,995374,996703,990992,997015,991467,992854,996353,998951,990585,994197,991504,997609,990535,993909,999026,996959,994595,997009,991173,991498,999089,996714,992189,996658,993562,995824,995034,998844\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 707\n", + "Returned IDs: 997485,997529,993939,998610,995861,996752,996328,990133,992450,993538,997103,997544,994830,991114,998710,993079,990359,994185,999372,995295,990889,991334,998484,994835,993127,997933,992434,998671,995211,995620\n", + "Ground Truth: 997485,997529,993939,998610,995861,996752,996328,990133,992450,993538,997103,996042,997544,994830,991114,998710,993079,994131,990359,994185,999372,995295,990889,991334,998484,994835,993127,997933,992434,998671\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 708\n", + "Returned IDs: 996052,996219,990448,999998,998464,991999,994964,991844,995581,990538,999876,997010,992604,990063,990366,996001,998593,999791,995596,992944,992337,992558,998495,991052,997470,995693,996959,996871,998758,990886\n", + "Ground Truth: 996052,996219,998383,990448,999998,998464,991743,995716,991999,994964,993412,991844,996232,995581,990538,999876,997010,992604,990063,991650,990366,996001,998593,999791,995596,992944,992337,995107,992558,990091\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 709\n", + "Returned IDs: 997103,999980,990537,999581,993939,999054,992448,991722,990220,990251,998910,994861,999279,990372,998733,997544,991389,993127,992449,993856,995268,995778,997176,993308,991507,994380,991664,992097,991560,996505\n", + "Ground Truth: 997103,999980,990537,998283,999581,993939,999054,992448,991722,996897,990220,990251,998910,994861,999279,990372,998733,997544,991389,993127,992449,993856,995268,995778,997176,993308,993046,993003,991507,994380\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 710\n", + "Returned IDs: 994922,991840,990958,992169,996001,998593,997822,990076,996351,992994,999729,996867,999676,990064,993143,992692,996854,997320,992944,990455,994380,992622,999453,991392,993168,998458,991015,995245,995638,992448\n", + "Ground Truth: 994922,991840,997591,990958,999261,992169,996001,998593,997822,990076,996351,992994,999729,990648,990399,996867,999676,991087,990064,993143,992692,996854,997320,992944,990866,990455,997620,994380,992622,996438\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 711\n", + "Returned IDs: 993698,991066,995007,992576,991011,997331,997168,995589,998723,996505,994191,996067,995245,994380,991857,992472,996277,998045,996716,995815,996062,990537,995907,994113,991747,996769,993561,993813,997675,991958\n", + "Ground Truth: 993698,991066,995007,992576,991011,997331,997168,995589,998723,996505,994191,996932,996067,995245,990951,994380,993778,991857,992472,996277,994419,995431,998045,996716,995815,996062,990537,995907,994113,991747\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 712\n", + "Returned IDs: 991720,990545,998170,993143,991999,993253,990432,999851,994964,995588,995515,995927,998294,997908,993484,997320,998550,992384,993885,994100,998101,999453,994561,997002,996001,992942,995400,994346,993877,991743\n", + "Ground Truth: 991720,990545,996296,998170,993143,999437,999764,991999,993253,991715,998562,994251,992979,990432,999077,999851,994964,995588,995515,995927,997876,998294,997908,993484,999343,997320,998550,992384,995748,993885\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 713\n", + "Returned IDs: 999384,994972,990096,995536,998697,997320,999898,999647,996001,996062,999144,997913,991154,995701,996219,995130,993504,998401,993809,997775,993906,996370,995596,993561,991459,995560,999653,997918,994964,991157\n", + "Ground Truth: 999384,994972,997811,990096,995536,998697,991179,997320,999898,999647,996001,996062,999144,997913,991154,995701,996219,995130,993504,998401,993809,992692,997775,993906,996370,992047,995596,993561,991459,995560\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 714\n", + "Returned IDs: \n", + "Ground Truth: 992669,990063,997526,998858,992604,999961,997110,992439,993115,995949,991309,998707,994724,994774,995982,993122,999478,990111,998383,990657,994569,999231,997205,991987,994454,997107,993976,998806,995455,992051\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 715\n", + "Returned IDs: 996042,997046,997822,993642,996328,997386,990137,993433,994419,996322,998545,997453,998165,996494,995289,998039,995200,997969,993561,993831,996867,995401,996062,991301,995685,998633,997289,994783,994742,994409\n", + "Ground Truth: 996042,997046,997822,993642,996328,997386,993827,990137,993433,993412,994419,996322,998545,997453,998165,996494,995289,998039,995200,997969,993561,993831,996867,995526,997732,994964,995401,993400,996062,991301\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 717\n", + "Returned IDs: 993179,992635,990697,991260,994886,991372,996884,996755,990532,995596,996001,992515,997550,992041,993820,990083,993071,998097,995515,996075,996805,999426,997002,990834,994862,997584,995358,993129,992733,994972\n", + "Ground Truth: 993179,992635,990697,991260,994886,991372,996884,996755,990532,995596,995622,996001,992515,997550,992041,993820,990083,990640,993071,998097,995515,996075,991142,996805,999426,997002,990834,994862,997584,991048\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 718\n", + "Returned IDs: 996536,993588,994897,994830,993342,997952,992664,998716,993335,998092,995097,998623,998903,998440,997296,998256,997073,998030,996183,992427,998097,994513,993866,990392,992458,994682,993715,994906,997002,996001\n", + "Ground Truth: 996536,993588,994897,994830,993342,997952,992664,998716,997301,993335,998092,995097,998623,998903,998440,997296,998256,997073,998030,990334,996183,992427,998097,994513,993866,990392,999963,992458,994682,992174\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 719\n", + "Returned IDs: 992330,997371,997913,995985,990397,990469,990560,998500,992384,992443,996193,990595,998921,995260,996840,994360,992546,999122,997320,999676,997984,997305,997831,993492,991626,996504,999603,995949,999165,998445\n", + "Ground Truth: 992330,997371,997913,995985,993072,990397,990469,990560,993048,998500,991468,990092,992384,992443,997889,996193,990595,998921,995260,996840,994360,992546,994281,999613,991631,999122,995365,997320,999676,997984\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 720\n", + "Returned IDs: 998555,998169,999781,990271,990096,997529,998122,999343,995638,992551,993800,997732,994145,992120,995535,999103,993637,996219,990089,999647,999319,990690,994532,998934,998314,990918,996328,990072,990366,999791\n", + "Ground Truth: 998555,998169,999781,990271,990096,997529,998122,999343,995638,992551,993800,997732,990091,994145,992120,995535,999103,993637,996219,995216,990089,999647,999319,990690,997918,994532,998934,997920,998314,990918\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 721\n", + "Returned IDs: 990242,998162,992937,990940,995444,993831,997046,996388,992160,997466,991163,991603,991973,993390,995702,998503,996867,997723,992521,998238,998267,998165,999595,997868,997822,993678,996769,990110,994954,991415\n", + "Ground Truth: 990242,998162,992937,990940,995444,993831,993799,997046,996388,992160,997466,991163,991603,991973,993390,996038,995702,998503,996867,997723,992521,998238,998267,998165,999595,992977,997868,998941,997822,993678\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 722\n", + "Returned IDs: 998536,993965,992407,999448,995461,998331,992052,999079,994119,995408,991707,997897,998051,996817,999212,996633,999433,996245,995110,996733,998177,992320,990551,998874,997881,991515,990169,994743,996134,991245\n", + "Ground Truth: 998536,993965,992407,999448,995461,998331,992052,999079,994119,995408,991707,997897,998051,996817,999212,991093,996633,999433,996245,995110,996733,998177,992320,990551,998874,997881,991515,990169,994743,999179\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 723\n", + "Returned IDs: 992636,995328,994522,999767,994742,992832,991850,991998,995012,992764,991007,992973,996911,998174,999759,990336,994835,992239,997153,994891,993747,990270,991629,998536,993621,993448,993959,996183,994175,996050\n", + "Ground Truth: 992636,995328,992933,994522,999767,994742,992832,996557,991850,991998,995012,992764,991007,992973,996911,998174,992779,999759,990336,994835,990364,992239,997153,994891,993747,990270,991629,998536,993621,993448\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 724\n", + "Returned IDs: 993775,998765,999791,995477,991921,990089,994742,997529,999058,996633,997401,995057,997698,996328,996302,999788,997309,999875,991588,990179,999192,999261,995438,994806,993053,993265,995289,995551,999138,991315\n", + "Ground Truth: 993775,998765,999791,995477,991921,990089,994742,997529,999058,996633,997401,995057,997698,996328,996302,999788,997309,999875,991588,990179,999192,999261,995438,994806,993053,993265,995289,995551,999138,995600\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 725\n", + "Returned IDs: 998168,992517,995768,995501,994356,995476,993755,990851,991382,991667,995479,992953,999425,990335,997340,995879,993319,991826,999130,998794,996184,994205,990145,998625,999107,999256,995073,995191,991779,992868\n", + "Ground Truth: 998168,992517,995768,995501,994356,995476,993755,990851,991382,991667,995479,999634,992953,999425,990335,995222,997340,995879,993319,991826,999130,998794,996184,994205,990145,998625,999107,999256,995073,995191\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 726\n", + "Returned IDs: \n", + "Ground Truth: 993035,995791,995004,995365,993570,993633,995588,999039,999592,993282,998837,990043,999620,993028,991999,997496,995155,994545,995657,994666,991407,991593,993505,993675,994543,994922,993708,993946,991196,990155\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 727\n", + "Returned IDs: 995865,990612,994611,996143,996318,991872,994214,997320,990210,997178,994812,992909,996805,997022,993885,998550,997305,991498,994964,999374,994699,994249,996755,992978,992066,998540,999982,994164,994778,992993\n", + "Ground Truth: 993990,995865,990612,994670,994611,996143,993745,996318,993154,991872,994214,997320,990210,997178,993579,994812,992909,996805,997022,993885,999351,998550,998287,997305,991498,990772,998344,994964,999374,994699\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 728\n", + "Returned IDs: 995932,993504,999516,990196,993071,993906,991440,991648,997555,991206,993515,992082,990350,997918,992571,993783,997790,992747,993696,993260,997002,992351,991435,995375,991994,996429,998726,996949,997965,993637\n", + "Ground Truth: 995932,993504,999516,990196,993071,993906,991440,997758,991648,997555,991206,994081,997135,995622,993515,990682,992082,993821,990350,997918,992571,993783,997790,992747,993696,993260,997002,994146,992351,991435\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 729\n", + "Returned IDs: 995655,998625,990154,993001,992110,992517,999555,990167,992752,992444,991667,998536,994465,999433,998006,993948,991620,990173,996408,998888,995879,999256,994423,995768,992595,993641,999319,991421,999994,995936\n", + "Ground Truth: 995655,998625,990154,993001,992110,992517,999555,990167,992752,992444,991667,998536,994465,999433,998006,993948,991620,990173,991188,990429,996408,998888,995879,994796,999256,994423,995768,993641,992595,999319\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 730\n", + "Returned IDs: 997025,995521,995907,998703,999653,995303,991161,992448,997251,995891,995537,993275,993313,993342,994380,990593,995778,994125,991844,997675,993561,996042,993127,998855,991459,991447,991794,995815,992576,994419\n", + "Ground Truth: 997025,995521,995907,998703,999653,995303,991161,992448,997251,995891,995537,997909,993275,993313,993342,994380,990593,995778,990725,994125,991844,997723,991338,997675,993561,996042,993127,998855,991459,991447\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 731\n", + "Returned IDs: 999374,999710,997320,997913,996504,992748,991860,999331,991801,998495,998540,993484,995942,991743,997301,996178,992147,995886,994964,993809,995961,991498,998246,990958,990957,998550,990210,996318,998029,991593\n", + "Ground Truth: 999374,999710,997320,997913,999701,991052,996504,992748,991860,999331,991801,998294,998495,998540,993484,995942,991743,995632,997301,996178,992789,992147,995886,994964,993809,995961,994850,991498,998246,990958\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 732\n", + "Returned IDs: 993561,996783,996313,996733,997023,996062,991958,993931,994444,990537,990096,998207,992322,999673,990382,991093,998536,994380,997453,998024,990899,990531,992752,995648,995521,993289,996001,998314,998416,997790\n", + "Ground Truth: 993561,996783,996313,996733,997023,996062,991958,993931,999436,994444,990537,990096,998207,992322,999673,990382,990072,991093,998536,994380,997453,995401,998024,990899,990531,992752,995648,995521,993289,996001\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 733\n", + "Returned IDs: 992037,998022,996277,998934,994906,996181,990400,996001,998720,998080,991459,994444,999020,992685,999026,999543,997204,993652,995780,994589,994424,992648,996379,992576,999759,998990,991248,991574,996861,992516\n", + "Ground Truth: 992037,998022,996277,998934,994906,995401,993880,996181,990400,996001,998720,998080,991459,999259,994444,999020,992685,999026,998221,999543,997204,994105,995091,996408,993652,995780,994589,994424,992648,996379\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 734\n", + "Returned IDs: 992712,995855,995356,998622,992163,996105,994167,990472,994955,993283,999216,997368,993300,998513,991232,992484,999041,993174,999219,998697,997977,999750,995672,998934,997417,995203,993918,998447,993260,998393\n", + "Ground Truth: 992712,995855,995356,995563,998622,992163,996105,994167,990472,994955,993283,999216,997368,993300,998513,991232,996144,992484,999041,993174,999219,998697,997977,990688,999750,995672,998934,997417,995284,995203\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 735\n", + "Returned IDs: 996196,997775,998950,994163,996140,994661,995596,997320,995560,990538,990532,996178,995983,990096,992124,990901,992558,991923,997913,997227,995552,995974,999564,996380,998697,996498,995289,995536,999639,998779\n", + "Ground Truth: 996196,997775,998950,995873,994163,996140,994661,995596,997320,995560,990538,992521,990532,996178,998417,991157,995983,990930,990096,992124,992277,990901,992558,991923,997913,997227,995552,997379,995974,999587\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 736\n", + "Returned IDs: 995507,999619,997832,996038,999530,993421,996323,996618,995905,995139,991027,995470,992448,996867,994409,997359,993400,995685,997046,996322,992130,997289,990836,992605,998045,992169,998520,997723,999482,997466\n", + "Ground Truth: 995507,999619,997832,996038,999530,993421,996323,996618,995905,995139,991027,995470,992448,996867,994409,997359,993400,995685,997046,996322,992130,997289,990836,992605,998045,993468,992169,998520,997723,999482\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 737\n", + "Returned IDs: 998682,999066,995372,996309,991787,995968,999544,993840,995250,999254,996452,993241,992944,991673,998849,995515,991248,997498,991650,992348,999453,997320,998921,997301,993447,998734,999686,991157,997453,996805\n", + "Ground Truth: 998682,999066,995372,996309,991787,995968,999544,993840,995250,999254,996452,993241,992944,991673,998849,995515,991248,997498,991650,992348,999453,997320,998921,994826,997301,997906,993447,998734,999686,995154\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 738\n", + "Returned IDs: 990115,998315,998925,992193,996431,995191,998055,994472,995223,990712,998671,996357,991682,994299,996016,992237,999205,991723,990187,997573,995789,999752,990091,996153,995919,991153,999832,992652,990931,996537\n", + "Ground Truth: 990115,998315,998925,992193,996431,995191,998055,994472,995223,990712,998251,998671,997954,996357,991682,994299,996016,992237,999205,999441,991723,990187,997573,995789,999752,996942,990091,997602,998084,998122\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 739\n", + "Returned IDs: 992932,990535,997304,998246,995981,998119,999761,992296,991900,998353,992815,997320,995473,991045,991534,992238,990588,993392,990949,998225,997353,992074,998950,990057,992357,999357,993809,992443,997913,995336\n", + "Ground Truth: 992932,990535,997304,996732,998246,995981,998119,999761,992296,991900,998353,992815,997320,995473,991045,995132,991534,992238,995792,990588,993392,993154,996950,992927,990949,997428,998225,998361,997353,992074\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 740\n", + "Returned IDs: 992862,998156,993256,994564,991428,996232,997173,990328,997374,992979,994742,996328,995819,997452,997309,999843,997918,995633,995295,998708,997914,990814,991189,996411,997208,993909,994382,990127,995274,999099\n", + "Ground Truth: 992862,998156,993857,993256,994564,994940,991428,996232,997173,990328,993440,997374,992979,994742,996328,995819,997452,997309,999843,997918,995633,995295,994850,998708,996752,997852,997914,990814,999453,998073\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 741\n", + "Returned IDs: 994672,992351,990885,993504,998732,996062,991425,998303,999811,990409,990196,997918,991037,997227,997977,995740,993289,994964,997790,990557,997002,995203,999093,990096,991212,999840,995932,994221,996196,992571\n", + "Ground Truth: 994672,992351,990885,993504,998732,996062,991425,998303,994468,999811,990409,990196,997918,991037,997227,997977,995740,993289,994964,997790,990557,993609,997002,995203,999093,990096,991212,999840,995932,993333\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 742\n", + "Returned IDs: 995231,997369,995168,994105,997666,992275,995385,990096,999176,996702,994779,995257,993866,997965,990658,992945,997847,998536,995393,997086,998177,992664,999129,996050,994834,991654,997142,995695,997576,997673\n", + "Ground Truth: 995231,997369,995168,994105,997666,992275,995385,990096,999176,996702,994779,995257,992987,993866,997965,990429,990658,999576,992945,991313,997847,998536,995393,997086,998177,992664,999129,996050,994834,990670\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 743\n", + "Returned IDs: 991630,997191,991140,998297,996734,993738,995401,990005,999530,992448,994319,998045,997868,998205,991969,995395,994019,991958,996277,997928,999820,992521,999731,995940,995813,995507,995815,995994,992919,999653\n", + "Ground Truth: 995121,991630,997191,991140,998297,992977,996734,995268,993738,995401,990005,999530,992448,994319,998045,997868,998205,995864,991969,995395,994019,997957,991958,993381,998346,996277,997928,999820,992521,999731\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 744\n", + "Returned IDs: 993885,998294,997320,990898,999374,993364,990917,991900,992896,993853,994964,990760,994476,996218,993484,990863,998899,998246,990296,992931,993662,990560,990146,991498,991593,990925,992546,999263,996279,999710\n", + "Ground Truth: 997006,993885,998294,997876,997320,990898,999374,996296,993364,990917,991059,991900,995266,992896,993048,993853,994964,994776,990760,994476,999328,996218,993484,999247,990863,998899,999647,999823,998246,990296\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 745\n", + "Returned IDs: 995793,991022,992518,995833,990862,998822,990535,996713,995771,998951,997976,994266,997320,994100,995336,998263,996714,998721,996961,992646,992183,992384,995438,999089,995279,997571,991889,993711,992609,992874\n", + "Ground Truth: 995793,991022,992518,995833,991782,990862,998822,990535,996713,995771,998951,997976,994266,997320,993735,994100,995336,998263,996714,998721,996961,992646,992183,992384,995438,999089,995279,997571,991889,999533\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 746\n", + "Returned IDs: 995030,990516,991590,998557,991968,992827,994716,994742,995633,999957,997662,996302,995289,994258,998196,992239,997319,992973,994893,992415,998999,991810,995200,993078,995012,998525,998543,999875,992364,999981\n", + "Ground Truth: 995030,990516,991590,998557,991968,992827,994716,994742,995633,999957,997662,996302,994317,995289,995239,994258,998196,992239,997319,992973,994893,992415,998999,991810,995200,993078,995012,998525,998543,999875\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 747\n", + "Returned IDs: 998132,992037,997320,996277,992294,996497,996247,991011,999820,990409,997918,998874,996368,991349,995130,993433,992979,996181,992759,999019,995873,992165,997027,996127,993652,997897,990899,990885,999026,992576\n", + "Ground Truth: 998132,992037,997320,993542,996277,992294,996497,995701,996247,991011,999820,990409,997918,994113,998874,996368,990442,991349,995130,993433,992979,996181,996294,999804,995876,991556,995047,992759,999019,990723\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 748\n", + "Returned IDs: 999453,991930,990772,996001,991248,991440,991609,993280,990314,995250,992395,995968,997332,991787,990399,991581,995372,990532,993032,994449,991158,994478,991542,991994,992182,992469,998567,999019,991537,994081\n", + "Ground Truth: 999453,991930,990772,996001,991248,991440,991609,993280,990314,995250,994484,992395,995968,997332,991787,998127,992884,990399,991581,995372,990532,993032,994449,999575,991158,994478,991542,991994,992182,992469\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 749\n", + "Returned IDs: 997032,995482,994925,996232,998221,996328,994742,999647,990096,993053,990435,994998,992450,996196,996178,998649,990887,998495,996001,999898,998067,996884,993783,991769,997126,998710,996062,993168,998550,996216\n", + "Ground Truth: 996111,997032,995482,994925,996232,998221,996328,994742,994798,999647,990096,993053,990435,994998,992450,996196,996178,998649,990887,998495,999586,996001,991743,997379,999898,998067,996884,993783,991769,997126\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 750\n", + "Returned IDs: \n", + "Ground Truth: 992701,995626,996082,991098,996599,995991,991236,995167,993375,994259,998389,990802,992437,995047,991327,990382,991741,990212,992247,998593,996001,997790,995568,997860,997494,991212,996733,997897,992862,991308\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 751\n", + "Returned IDs: 997015,991568,993168,993050,998992,990535,992744,995336,993562,996714,998951,995252,992646,999531,997082,994964,999026,995824,992911,992331,999144,994197,998495,999437,991444,997709,990925,993484,990797,997918\n", + "Ground Truth: 997015,991568,993168,993050,998992,990535,992744,995336,993562,996714,998951,995252,992646,999531,997082,994964,999026,990936,995824,992911,995536,992331,993648,999144,994197,998495,999437,991444,997709,990925\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 752\n", + "Returned IDs: 994259,995626,998376,994280,997790,998639,999093,995167,993262,993024,990212,991212,995343,997897,997534,990877,992864,995047,992168,996767,996216,992701,993187,995991,996599,991994,998032,992780,991098,991661\n", + "Ground Truth: 994259,995626,998376,994280,997790,996424,998639,999093,995167,993262,993024,990212,991212,995343,997897,997534,990877,992864,994815,995047,992168,990494,996767,997494,996216,992701,995705,993187,995991,996599\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 753\n", + "Returned IDs: 994140,993390,993831,991958,997046,991415,996042,998776,996719,995289,993561,994409,997822,995052,994783,993813,999099,991790,990940,996322,996388,998633,997554,998165,996867,997630,996924,993738,995679,994743\n", + "Ground Truth: 994140,993390,993831,991958,997046,991415,996042,998776,996719,995289,993561,994409,997822,995052,994783,990193,993813,999099,991790,990940,995873,996322,996388,998633,993856,997554,998165,996867,997630,996924\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 754\n", + "Returned IDs: 998950,996805,992944,993696,992041,998180,999210,999426,997470,991650,997371,995633,997403,997913,993504,994742,999676,998495,998855,999639,993747,993529,998790,992520,996196,996219,990096,996397,999367,998913\n", + "Ground Truth: 998950,996805,999647,992944,993696,992041,998180,997906,999210,999426,994089,997470,991650,997371,995633,997403,997913,993504,994742,997291,995401,999676,998495,998855,999639,993747,993529,998790,992520,996196\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 755\n", + "Returned IDs: 993791,999247,997320,998302,998090,996296,997661,993260,999290,994360,991774,998246,997470,993845,998337,997371,996592,990949,991743,992950,990231,997913,993741,995949,995260,997586,995596,993484,992896,990957\n", + "Ground Truth: 993791,999247,997320,998302,998090,996296,997661,993260,999290,994360,991774,998246,997470,993845,998337,990467,997371,996592,990949,991743,992950,990231,997913,993741,995949,995260,997586,995596,993484,993143\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 756\n", + "Returned IDs: 991507,991591,999957,998162,995778,999054,995958,997723,994861,998910,993209,997700,997020,999137,992448,991401,990105,994013,997046,999415,991671,995665,995493,996079,996867,998267,999787,997544,992002,999646\n", + "Ground Truth: 991507,991591,999957,998162,995778,999054,995958,997723,994861,998910,993209,997700,997020,999137,990836,992448,991401,999412,993899,990105,994013,997046,999415,991671,995665,995493,998609,996079,996867,998267\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 757\n", + "Returned IDs: 992372,995313,997856,998063,993335,997340,991620,998379,994682,995299,992026,990712,999381,991382,992882,990851,990173,999798,999348,995768,998625,997527,990579,994356,995501,992953,991667,992832,990392,996301\n", + "Ground Truth: 992372,995313,997856,998063,993335,997340,991620,998379,994682,995299,992026,990712,999381,991382,992882,990851,990173,999798,999348,995768,998625,995222,996037,997527,990579,994356,995501,992953,991667,992832\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 758\n", + "Returned IDs: 996001,991119,995455,995755,999009,998861,997337,990862,998122,990096,996809,994879,995701,990866,995425,999480,998514,999388,997586,995401,990091,991459,999289,991154,991903,995536,997305,999663,999898,990974\n", + "Ground Truth: 996001,991119,995455,995755,999009,998861,991881,999916,997337,990862,990505,998122,990096,996809,996406,994879,995701,990866,995425,999480,996408,998514,999388,997586,995401,990091,995132,991459,999289,991154\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 759\n", + "Returned IDs: 994833,998959,993560,994834,999373,995393,996797,998623,999136,993055,995237,998787,993366,998831,997210,993948,995936,998383,993014,990952,991844,998758,998565,995780,998861,999724,995695,990005,992903,992636\n", + "Ground Truth: 994833,998959,993560,994834,999373,995393,996797,998623,999136,993055,995237,998787,993366,998831,997210,993948,995936,998383,993014,990952,994906,991844,998758,998565,995780,998861,999724,995695,990005,992903\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 760\n", + "Returned IDs: 990061,990273,995309,993583,990206,994254,993500,996380,997547,992052,990042,994679,993077,995459,999022,993811,991977,991432,997866,995728,993742,990776,996328,992558,990894,993879,994280,997494,999639,994891\n", + "Ground Truth: 990061,990273,995309,993583,990206,994254,993500,996380,991064,997547,992052,990042,994679,993077,995459,999022,993811,991977,991432,997866,995728,993742,990776,996328,995600,992558,990894,993879,994280,997494\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 761\n", + "Returned IDs: 990127,990331,993209,995778,992862,994409,995268,990887,995536,999179,993313,994488,992260,990975,996328,994954,990930,992052,994835,992636,993809,992595,991958,993561,992917,996001,996232,994742,995907,990593\n", + "Ground Truth: 990127,995198,990331,993209,995873,995778,992862,994409,995268,995401,990887,995536,999179,993313,994488,992260,990975,997184,996328,994954,992293,990930,992052,994835,992636,993809,992595,995122,999581,991958\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 762\n", + "Returned IDs: 997550,995596,994618,998921,999338,999426,996075,997661,997113,991992,995033,992041,995515,996318,993785,992944,997584,996516,994971,997984,991984,994233,996741,996631,996387,999698,999588,995467,992636,997291\n", + "Ground Truth: 997550,995596,994618,998921,999338,999426,996075,997661,997113,991992,995033,992041,995515,997448,996318,993785,992944,997584,996516,994971,997984,991984,993125,995401,994233,996741,996631,996387,999698,999588\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 763\n", + "Returned IDs: 995317,993678,996194,992992,997854,993256,994541,993044,990987,997251,990069,999287,995159,996438,994064,997806,993621,993014,992313,996232,991285,994639,999179,990776,991172,993948,994169,993039,990953,991319\n", + "Ground Truth: 995317,993558,993678,996194,992992,997854,993256,994541,991075,993044,990987,997251,990069,995401,999287,997429,995159,996438,991863,994064,999386,997806,994428,993621,993014,992313,996232,991285,993454,997770\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 764\n", + "Returned IDs: 994955,998933,993194,999798,991685,993665,998536,997944,992825,994280,995168,995932,992561,993715,995589,997453,992823,995945,993931,993037,994167,990387,999840,990364,999543,991712,995834,997506,990409,990658\n", + "Ground Truth: 994955,998933,993194,999798,991685,993665,999052,998536,997944,992825,994280,995168,997155,992850,995932,992561,993715,995589,995366,997453,992823,998778,995945,993931,997926,993037,994167,990387,999840,990364\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 765\n", + "Returned IDs: 994557,991058,997638,993929,994119,995332,990345,992704,997822,997309,990987,990678,999210,997693,991107,995620,999643,990226,992147,994280,997945,994336,997126,992962,994541,991177,999612,991378,991848,994142\n", + "Ground Truth: 994557,991058,997638,997807,993929,994119,995332,990879,990345,992704,997822,997309,990987,993014,990678,999210,997693,996071,991107,995620,999643,990226,992147,994280,997945,994336,997126,992962,994541,991177\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 766\n", + "Returned IDs: 992293,990662,990771,998878,998017,998549,996147,996657,995158,999302,993848,996001,996759,999853,999922,997554,990811,999342,990573,992862,994223,991588,993344,996368,990211,997079,995575,998710,991585,993168\n", + "Ground Truth: 992293,990662,990771,998878,998017,998549,996147,996657,995158,999302,993848,996001,996759,995002,999853,999922,997554,990811,999342,990573,992862,994223,991588,999587,993344,993374,996368,990211,997079,995575\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 768\n", + "Returned IDs: 999117,994643,993122,993608,995625,992677,999155,993232,990361,991987,996395,998078,999231,991630,997149,995807,999834,996809,993795,998257,997110,994092,997752,997568,995373,998536,995105,992288,993076,992685\n", + "Ground Truth: 999117,994643,993122,993608,995625,992677,999155,993232,990361,996395,991987,998078,999231,991630,997149,995807,995401,999834,991052,996809,993795,998257,997110,994092,992206,997752,997568,995373,998536,995105\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 769\n", + "Returned IDs: 999388,990154,998971,999663,994636,999555,992282,992403,998701,991596,996406,997803,992155,993132,991385,999153,994181,990394,992716,994335,999898,994834,990305,994638,993458,994155,993560,999677,991154,997154\n", + "Ground Truth: 999388,990154,998971,999663,994636,999555,992282,992403,998701,991596,996406,992445,997803,992155,993132,991385,999153,994181,990394,992716,994335,999898,994834,990305,994638,993458,994155,993560,999677,991154\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 770\n", + "Returned IDs: 991889,995949,998294,995260,996053,998692,999374,991705,991406,992309,995928,993143,997627,996334,995693,994919,990432,996854,995354,992169,997429,990760,999432,990146,990958,998170,992252,994195,990189,996351\n", + "Ground Truth: 991889,995949,998294,995260,996053,998692,999374,991705,991406,996554,993837,992309,990026,995928,993143,997627,996334,992979,995693,994160,994919,990432,990545,990103,993412,996854,990490,995354,996202,992169\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 771\n", + "Returned IDs: 993416,992684,994855,991958,998158,998382,990794,993561,994987,996040,996062,990254,996371,998593,995923,995547,993738,994076,996834,994419,997453,999868,992260,998297,995373,998891,997406,998416,995401,996198\n", + "Ground Truth: 993416,992684,994855,991958,998158,998382,990794,993561,994987,996040,996062,990254,993673,991052,996371,999236,998593,992017,995923,994946,995547,993738,994076,996834,998323,994419,997453,999868,992260,998297\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 772\n", + "Returned IDs: 997002,998902,994906,992790,994334,995894,997301,996658,999442,998951,995560,991621,994661,992239,996001,995401,992165,995836,992759,996220,995357,997918,999587,994088,992641,995536,990561,991485,997320,994946\n", + "Ground Truth: 997002,998902,994906,992790,994334,995894,997301,996658,999442,998951,995560,991621,994661,992239,996001,995401,992165,995374,995836,992759,996220,994362,993820,995357,998045,997918,991989,990140,999587,994088\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 773\n", + "Returned IDs: 992041,996075,990022,990366,993785,992167,990697,991372,995515,994440,999426,991984,996001,991109,999473,997661,995596,992872,998498,996219,993484,992733,992927,999053,990934,992617,996996,999806,990957,993807\n", + "Ground Truth: 992041,996075,990022,990366,993785,992167,990697,991372,995515,994440,999426,991984,996001,991109,999473,997836,994776,998654,997661,993928,995596,996227,992872,998498,996219,993484,992733,992927,999053,990934\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 774\n", + "Returned IDs: 992383,990383,991550,990884,999387,999451,990376,994964,996894,995983,992824,996196,998600,998401,990538,993781,995427,995372,999639,993706,997903,994979,994852,995459,991817,993492,992190,996085,998697,995841\n", + "Ground Truth: 992383,990383,991550,990884,999387,996445,999451,990376,991923,994964,996894,995983,992824,996196,998600,998401,990538,993781,995427,995372,999639,993706,997903,994979,994852,995459,991817,993492,992190,998545\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 775\n", + "Returned IDs: 990832,995100,990194,993435,997725,997331,996475,996809,996219,996830,999435,995880,995131,999331,996216,993342,998084,998122,999430,999388,999647,992100,997047,992237,999898,993148,993391,993931,998387,990957\n", + "Ground Truth: 990832,995100,990194,993435,997725,997331,996475,996809,996219,996830,992892,999435,991187,995880,991459,995131,999331,999384,996067,996216,993342,998084,998122,995130,999430,999388,999647,993192,996913,998925\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 776\n", + "Returned IDs: 999159,991582,995632,997284,997354,991348,996296,997320,992691,993018,991763,993454,999777,995194,991879,996199,996387,991131,992896,990456,996075,996626,999658,994415,990375,998950,997370,994341,997315,993253\n", + "Ground Truth: 999159,991582,995632,997284,997354,994225,991348,995878,992022,996296,997320,992691,993018,991763,991992,993454,999777,995194,991879,996199,996387,991131,992896,990456,991662,996075,995108,996626,999658,994415\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 777\n", + "Returned IDs: 999663,995536,992225,998514,998861,999009,999711,990096,998720,991459,992293,996809,996001,992553,993139,994434,995701,992862,998200,999216,998082,997586,993948,997506,991893,998495,996996,991154,990786,999898\n", + "Ground Truth: 999663,995536,992225,998514,998861,999009,999711,990096,998720,991459,992293,996809,996001,992553,993139,994434,995401,993821,995701,992862,998200,996406,999216,998082,998568,997586,993948,995961,997506,991893\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 778\n", + "Returned IDs: 994401,993039,998972,995065,992985,993931,995159,990184,998851,998566,996277,993799,994835,992576,997181,995648,995760,996223,995268,993256,996505,999759,995790,995815,993879,997126,990956,990391,995685,996173\n", + "Ground Truth: 994401,993039,997077,998972,995065,992985,993931,995159,990184,998851,998566,996277,993799,994835,992576,997181,995648,995760,996223,995268,993256,996505,999759,990291,990942,995790,995815,993879,997126,990956\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 779\n", + "Returned IDs: \n", + "Ground Truth: 998496,995840,994879,996075,991166,992442,991984,993834,997258,992041,998708,993902,998200,997586,997305,997770,993766,992330,996230,993174,993809,996933,991476,995701,994558,996001,999701,996657,993696,999041\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 780\n", + "Returned IDs: \n", + "Ground Truth: 998428,996270,991605,997151,990010,999139,994196,993715,993181,991633,996483,990705,990712,994101,995421,994280,995349,999435,992779,993460,990998,993347,994351,998250,992441,994510,994193,991212,999205,997733\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 781\n", + "Returned IDs: 994273,994098,999430,998671,999144,990711,993100,996816,991840,998416,993161,996222,994801,991672,999225,997331,996328,999532,991378,993501,993127,990392,997193,993929,994351,990226,993805,997967,990408,990340\n", + "Ground Truth: 992226,994273,990815,994098,999430,998671,995704,999144,990711,993100,999435,996816,994558,991840,997955,998416,993725,993161,996222,994801,991672,999225,997331,996328,999171,998142,991938,999532,991378,998729\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 782\n", + "Returned IDs: 999530,998070,992448,995685,991301,994409,996323,991699,994002,996987,998714,998545,997046,997675,995507,992605,999578,999043,996322,991671,996607,991247,999884,991583,996042,995139,993561,998910,998045,996390\n", + "Ground Truth: 999530,998070,992448,995685,991301,994409,996323,991699,994002,996987,998714,998545,997046,997675,995507,992605,999578,999043,996322,993412,991671,996607,998697,991247,990337,999884,991583,993400,996042,995401\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 783\n", + "Returned IDs: 990089,995454,997292,995659,999791,995045,993637,992334,999444,990918,996147,998076,998097,992636,998302,999302,995306,992415,996516,997490,991144,996001,997349,997933,995295,995345,999099,999338,997533,998921\n", + "Ground Truth: 990089,995454,995720,997292,995659,999791,995045,993637,999858,992334,991537,999444,990918,996147,998076,998097,992636,998302,999302,995306,992415,996516,997490,994373,995133,991144,996001,995755,993663,997349\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 784\n", + "Returned IDs: 998256,999852,990998,998279,996986,996147,993335,996189,999342,999334,998343,995326,999273,999924,998959,996960,997790,994830,990402,994185,992636,990771,990956,998335,994891,990434,992664,996328,990027,996594\n", + "Ground Truth: 998256,999852,990998,998279,996986,991347,996147,993335,996189,999342,999334,998343,995326,999273,999924,999555,998959,996960,997790,998122,994830,990402,998680,994185,997506,993545,992636,990771,990956,997494\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 786\n", + "Returned IDs: 990635,998143,998205,993672,996187,997868,995864,994954,998941,999926,992521,991969,998587,991892,993269,990341,995814,999250,998527,996256,990603,996987,994409,997909,997466,992448,996169,998278,992426,996162\n", + "Ground Truth: 990635,998143,998205,993672,996187,997868,995864,994954,998941,999926,992521,991969,998587,991892,993269,990341,995814,999250,998527,996256,990603,996987,994409,997909,997466,992448,996383,996169,998278,992426\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 787\n", + "Returned IDs: 994670,991511,996219,992630,995274,990679,992931,992124,996714,990271,992277,993154,994778,999243,997976,990910,997404,992748,999596,991456,995981,995502,990448,994894,992134,997571,997348,998337,999804,993342\n", + "Ground Truth: 994670,991511,996219,993517,992630,995274,990679,994916,990909,992931,992124,999723,996714,990271,991085,992277,993154,994778,999243,997976,990910,995161,999340,997404,992748,999596,991456,995981,993481,997018\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 788\n", + "Returned IDs: 997411,994730,992293,997436,993725,997925,997529,990435,999922,994606,992605,992582,995701,993342,996368,999740,998000,997723,996028,990811,995435,992024,995016,998934,992473,994382,994636,993168,996001,990012\n", + "Ground Truth: 997411,994730,992293,997436,993725,990815,997925,997529,990435,999922,994606,992605,998105,992582,993856,995701,993342,996368,995906,999740,998000,997723,991301,994674,996028,994728,990811,998708,995435,998181\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 789\n", + "Returned IDs: 999240,993473,997195,993986,992558,999860,993811,991595,991550,993077,995552,991025,992121,998929,990078,999479,999715,996750,993158,995967,996196,990249,997054,991426,994679,992375,999400,992161,999540,996523\n", + "Ground Truth: 999240,993473,997195,993986,992558,999860,993811,991595,991550,993077,995552,991025,992121,998929,990078,995972,999479,999715,996918,991817,996750,993158,993400,995967,996196,990249,999631,994168,997054,991426\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 790\n", + "Returned IDs: 992147,994964,994850,995516,999210,995927,995181,997305,996219,999374,998302,995132,994778,994584,990843,991650,998123,994789,991999,995755,998950,996075,991801,996504,991348,996186,996805,991498,997831,995656\n", + "Ground Truth: 992147,994964,994850,995516,998464,998287,999210,995927,993154,995181,997305,996219,991582,999374,998302,995132,994778,994584,990843,991650,998123,994789,991999,995755,998950,996075,997906,991801,996504,991348\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 791\n", + "Returned IDs: 996633,992598,994639,996155,999059,999192,994541,998761,995551,990392,998348,993825,997026,994169,996960,996194,994356,993448,993314,994742,993691,992832,994327,993725,992595,994861,995122,999774,999225,997340\n", + "Ground Truth: 996633,992598,994639,996155,999059,999192,994541,998761,995551,990392,998348,993825,997026,994169,996960,992992,996194,997838,994356,993793,993448,993314,995401,994742,993691,992832,992540,994327,993725,992595\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 792\n", + "Returned IDs: 998902,998236,991131,998967,996030,994604,991999,998001,993399,995515,997918,998492,995357,998734,990176,999698,991157,992944,994329,995865,995988,994789,991609,995372,991801,993244,999453,990843,994216,992510\n", + "Ground Truth: 998902,998236,991131,998967,996030,994604,991999,991359,990444,998001,993399,995515,991499,997918,998492,995357,998734,990176,999698,992839,994449,991157,997831,992944,994329,990219,995865,995988,995364,994789\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 793\n", + "Returned IDs: 995047,997790,996001,995184,998593,999827,990399,993289,993280,990364,992862,994743,996657,998697,991212,994280,998250,999453,997897,997002,990096,992395,996043,994742,996062,996368,999019,998303,993931,995536\n", + "Ground Truth: 995047,997790,996001,995184,998593,999827,990399,993289,993280,990364,992862,995401,996657,994743,998697,991212,994280,998250,996277,999453,997897,997002,990096,992395,996043,994742,997494,996062,996368,999019\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 794\n", + "Returned IDs: 998679,990930,991585,993293,993813,998779,995633,995289,991141,994742,994730,994140,994436,993997,999402,998980,990435,993856,994409,991664,993831,995702,995648,995216,997554,995052,999163,990940,995268,995016\n", + "Ground Truth: 998679,990930,991585,993293,993813,998779,995633,995289,991141,994742,994730,994140,994436,993997,999402,998980,990435,993856,994409,991664,993831,995702,995648,997723,990361,995216,997554,999805,995052,996277\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 795\n", + "Returned IDs: 997586,995887,994832,990605,992068,996429,997379,996217,999711,993260,990866,993114,992112,997334,992983,990547,999284,992661,991880,999176,996105,996305,995430,999210,994558,999430,998513,994247,998359,995672\n", + "Ground Truth: 997586,995887,994832,990605,992068,999467,996429,997379,996217,996901,999711,993260,990866,993114,999216,992112,997334,992983,994200,990547,999284,992661,991880,999176,996424,996105,996305,995321,995430,998825\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 796\n", + "Returned IDs: 995252,992932,997320,997483,990231,997771,992228,995336,992692,991900,992330,998090,992069,992969,998951,990885,998246,998365,996296,993696,991131,990172,992978,995802,996196,992942,995942,996646,994757,993529\n", + "Ground Truth: 995252,992932,997320,997483,992096,990231,990541,997771,992228,995336,992277,992692,991900,992330,998090,992069,992969,998951,990885,998246,998365,998653,996296,993696,991131,990172,992978,995802,996196,996246\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 797\n", + "Returned IDs: 990050,996825,999206,993165,992358,996087,998507,993155,994051,998267,996918,997420,991319,997806,997438,996394,999296,990116,995516,997752,999881,998537,991799,993727,998778,999646,993453,991107,999871,997942\n", + "Ground Truth: 990050,996825,999206,993165,992358,996087,998507,993155,994051,998267,996918,997420,991319,997806,997438,999105,998021,996394,999296,990116,990386,994301,995516,997752,999881,998537,991799,997416,993727,998778\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 798\n", + "Returned IDs: 992832,995551,996633,994806,995326,992598,997814,999126,997026,993314,996155,994541,998348,993448,990127,993715,991850,997838,998536,991712,994908,994835,996328,993014,997340,998073,995159,995122,993046,997282\n", + "Ground Truth: 992832,995551,996633,994806,995326,992598,997814,999126,997026,993314,995401,996155,994541,998348,993448,990127,993715,991850,997838,998536,991712,994908,994835,996328,993014,997340,990956,998073,993560,995159\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 799\n", + "Returned IDs: 990941,996809,993727,990255,991471,999158,995963,998536,991890,996930,990930,999906,990194,998250,993232,992809,995144,994351,999706,996797,990116,994431,995701,990521,990956,993293,993980,996408,991245,996531\n", + "Ground Truth: 990941,996809,993727,990255,996043,991471,999158,990361,995963,998536,991890,996930,990930,997723,999906,990194,998250,993232,992809,995144,994351,999706,996797,993137,990116,994431,995701,990521,990956,993293\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 800\n", + "Returned IDs: 993708,995357,992134,992692,997918,998967,993145,991015,996592,993696,991131,999077,995336,993492,993781,990957,993512,998245,999144,996075,993683,999784,995210,996840,996085,994552,993529,998734,990949,994046\n", + "Ground Truth: 993708,995357,992134,993633,992692,997918,998967,993145,998996,991015,996592,993696,991131,998011,999077,995336,995693,993492,993781,992946,990957,993512,998245,999144,996075,992277,993683,997470,999784,995210\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 801\n", + "Returned IDs: \n", + "Ground Truth: 995203,993053,990918,993561,996645,990096,998416,995283,995873,990271,991464,994558,997370,994145,999653,994742,996147,993154,995310,997906,992334,998382,991168,994853,991565,999806,992408,995756,997920,990827\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 802\n", + "Returned IDs: 998500,991720,998921,998839,996218,994281,995181,996504,996890,992147,999263,995212,996785,993048,998029,993648,993443,991743,999374,995985,997470,995372,997371,994850,996805,997301,994497,992789,999483,995884\n", + "Ground Truth: 998500,991720,995621,998921,998839,996218,994281,995181,996504,996890,992147,999263,995212,996785,993048,990362,998029,993648,993443,991743,999374,991626,995985,997470,995372,994179,997371,994850,996805,997301\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 803\n", + "Returned IDs: 992321,995404,994163,990159,996196,993612,991009,999564,990548,996766,996690,994384,994950,996637,994957,992608,992215,995013,996127,992558,993670,992661,997775,994912,998220,996963,997902,990303,991550,994000\n", + "Ground Truth: 992321,995404,994163,990159,996196,993612,992458,991009,999564,990548,996766,996690,994384,994950,991601,996637,994957,992608,992215,995013,996127,992558,993670,999639,992661,997775,994912,998220,996963,992650\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 804\n", + "Returned IDs: 997309,997914,990705,992169,998123,990776,999273,993677,994119,998495,996328,994765,994280,992364,996001,999179,998180,990328,993993,991882,994817,992944,991334,994310,994835,996809,994587,997480,995295,995326\n", + "Ground Truth: 997309,997914,990705,992169,991720,998123,990776,999273,993677,994119,998495,996328,994765,994280,992364,996001,999179,995372,998180,990328,993993,991882,998466,994817,992944,991334,994310,994835,996809,994587\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 805\n", + "Returned IDs: 991642,990011,993906,995295,993725,994879,995852,997002,994689,991315,997790,996811,995823,998359,996328,995586,997918,999500,994307,994012,994742,990532,998067,999025,997379,990600,996132,990174,994817,999891\n", + "Ground Truth: 991642,990011,993906,995295,991435,993725,994879,995852,997002,994689,991315,998848,997790,992161,996811,990954,994206,994964,995823,992226,998359,991989,996328,995586,998105,990494,997918,999500,994307,994012\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 806\n", + "Returned IDs: 998343,990976,992056,995185,990255,990449,993875,995695,992293,998831,999334,995672,994510,998453,990399,999498,998589,999663,991836,999087,999502,997790,996082,994815,991038,992862,990959,993515,994365,990292\n", + "Ground Truth: 998343,990976,992056,995185,990255,990449,993875,997088,991256,995695,992293,998831,999334,995672,993545,994510,998453,990399,998834,999498,998589,999663,998959,991836,999087,999502,997790,996082,995074,994815\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 807\n", + "Returned IDs: 993052,999453,996333,999122,992571,993260,990196,998123,996193,997918,999210,991713,993504,993080,992944,993071,992169,994107,993143,990882,995203,993355,998180,997002,996981,994200,997370,998950,993280,993781\n", + "Ground Truth: 993052,999453,996333,999122,992571,993260,990196,998123,996193,997918,999210,991713,993504,993080,990098,992944,993071,992169,994107,993143,991052,990882,995203,993355,998180,996676,997002,994162,996981,994200\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 808\n", + "Returned IDs: 999707,991783,997331,993192,996475,990039,995298,991382,997481,993617,994558,999430,997436,996830,999435,994830,993435,992628,991595,997442,996495,994098,994113,999556,995342,998926,997270,994897,992479,990377\n", + "Ground Truth: 999707,991783,997331,993192,996475,990039,995298,991382,997481,993617,990640,994558,999430,997436,990527,996830,999435,994830,993435,992628,996219,993566,991595,997442,996495,993260,994098,994113,999556,995342\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 809\n", + "Returned IDs: 994964,996219,991582,993168,993289,999827,995927,997370,993143,997822,999104,997732,997918,997586,994922,990866,994280,990862,990146,999343,992169,992147,996001,995536,990382,990096,990224,996062,997002,997284\n", + "Ground Truth: 994964,996219,991582,993168,993289,999827,995927,997370,993143,997920,997822,999104,991052,997732,993412,997918,997586,994922,990866,994280,995210,990862,990146,999343,992169,992147,997028,999289,996001,995536\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 810\n", + "Returned IDs: 998314,999831,995385,993086,990096,998536,996918,992407,998177,998405,999287,993965,999079,994661,999194,992747,991052,996733,991093,998968,991724,991243,993239,999320,991571,993609,993715,997897,999384,990583\n", + "Ground Truth: 998314,999831,995385,993086,990096,998536,996918,992407,998177,998405,999287,993965,999079,994661,999194,992747,991052,996733,991093,998968,991724,991243,993239,999320,991571,993609,997775,993715,997897,999384\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 811\n", + "Returned IDs: 992215,998213,992615,998827,994287,997859,996196,990068,991550,994163,995886,992747,997124,995552,996442,992558,990343,995372,995983,999639,993693,992170,991817,993130,997258,999385,993260,996127,996404,990620\n", + "Ground Truth: 992215,998213,992615,998827,994287,997859,996196,990068,991550,994163,995886,992747,997124,990486,995552,996442,992558,990343,995372,995983,993693,999639,992170,990910,991817,993130,997258,999385,993260,996127\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 812\n", + "Returned IDs: 990925,994160,996343,990958,998797,999262,995884,992546,995947,990535,991396,995485,990461,992994,993083,990146,992284,997102,991444,998458,995771,993562,999600,990936,994922,994685,998927,999644,997192,996959\n", + "Ground Truth: 990925,994160,996343,990958,998797,999262,995884,992546,995947,990535,991396,995485,990461,992994,993083,990146,992284,997102,997876,991444,998458,995771,993562,999600,993914,997053,990936,994922,994685,998927\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 813\n", + "Returned IDs: 994334,999144,997012,994819,994088,996658,995374,999820,998011,996903,995279,990215,997571,997918,999089,996425,992087,997910,999374,997002,996282,998385,995453,991131,991977,991074,993407,992147,990275,999026\n", + "Ground Truth: 994334,999144,997012,994819,994088,996658,995374,999820,998011,996903,995279,990215,997571,993868,997918,999089,996425,992087,997910,999374,993412,997002,996282,998385,995453,999562,991131,991977,991074,993407\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 815\n", + "Returned IDs: 991269,990392,995283,992713,994853,999126,998811,991168,993678,991189,995295,993561,991464,994742,996411,996752,991769,993813,997141,991789,997452,993725,995633,999496,991981,996628,993209,994090,994119,992832\n", + "Ground Truth: 991269,990392,995283,997682,992713,994853,999126,998811,991168,993678,991189,995295,993561,991464,994742,997933,996411,996752,991769,994386,993813,997141,996055,991789,990614,997452,993725,999496,995633,991981\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 816\n", + "Returned IDs: 993392,992027,995771,993620,997984,996504,998921,996218,993518,997704,994281,994939,991782,998692,995985,994922,998822,992709,991889,999374,990595,997192,992546,994775,992147,993161,999327,994641,995554,994630\n", + "Ground Truth: 993392,992027,995771,993620,997984,996504,998921,996218,993518,997704,994281,994939,991782,998692,993457,995985,991873,994922,998822,992709,991889,992932,999374,990595,997192,997006,992546,994775,992147,993161\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 817\n", + "Returned IDs: 994304,997868,997586,992521,995814,994636,994954,996918,992293,997723,993438,997466,998260,995701,999731,991790,996386,998255,993831,997305,998346,990097,991967,990078,996187,997216,995963,996368,993053,991163\n", + "Ground Truth: 994304,997868,997586,992521,995814,994636,994954,996918,992293,997723,993438,997466,998260,995701,990866,999731,991790,996386,998255,993831,997305,998346,995873,990097,991967,990078,996360,996187,994419,998564\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 818\n", + "Returned IDs: \n", + "Ground Truth: 992909,994214,995090,997320,998808,998090,992978,994290,991479,999621,990588,994427,991883,995372,995802,998364,992692,995826,995961,995228,994775,993662,992069,991503,991873,992969,994280,999761,997913,998821\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 819\n", + "Returned IDs: 992293,995701,993848,993834,992225,998549,996657,992862,998593,992367,990811,995158,990255,998886,990211,998200,994809,994587,998878,992773,993937,998215,999853,990662,990573,990399,990097,997687,999977,994409\n", + "Ground Truth: 992293,995701,993848,993834,992225,998549,996657,992862,998593,992367,990811,995158,990255,998886,990211,998200,994809,994587,998878,992773,993937,991603,998215,999853,990662,992473,990573,990399,991630,990097\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 820\n", + "Returned IDs: 997331,998147,994113,995382,992433,993281,998770,996702,999643,991011,996494,996722,993778,993939,996067,995620,991148,998484,996219,991179,994692,997837,996080,991377,994589,994382,993936,996371,996475,995032\n", + "Ground Truth: 997331,998147,994113,995382,992433,993281,998770,996702,999643,991011,996494,996722,993778,993939,996067,996866,995620,992840,991148,998484,990527,996219,991179,994692,994984,995401,992987,997837,996080,991377\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 821\n", + "Returned IDs: 999837,992752,991633,998594,991552,991605,998279,997673,993611,998389,997151,996082,995421,998108,992864,993715,999139,997050,995705,995626,994734,996216,995232,994196,993347,996270,991469,993001,997897,996328\n", + "Ground Truth: 999837,992752,991633,998594,991242,991552,990133,991605,998279,997673,993611,998389,997151,995167,996082,995421,998108,992864,990950,993715,998716,999139,997050,995705,995626,990017,996216,994734,995232,994196\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 822\n", + "Returned IDs: 991765,993319,999798,993396,996914,997340,996535,995768,998525,990408,997714,998168,994356,999601,991779,992250,995501,992517,994841,993993,996918,999886,999052,999496,998224,991712,991850,992561,999256,992868\n", + "Ground Truth: 991765,993319,999798,993396,996914,997340,996535,995768,998525,990408,997714,998168,994356,999601,990183,991779,997925,995222,992250,993817,995501,992517,994841,992131,993993,996918,995755,990711,999886,999052\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 823\n", + "Returned IDs: 991356,992558,991977,997547,993255,990602,990226,994560,999908,993583,990894,999305,993077,997859,992113,993742,992827,993839,994472,998892,996318,995886,995552,995542,999026,990279,997918,998495,992824,994254\n", + "Ground Truth: 991356,992558,991977,997547,993255,990602,990226,994560,999908,993583,997806,990894,999305,993077,997859,992113,993742,998151,992827,993839,994472,990995,998892,995374,996318,995886,995552,995542,999026,990279\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 824\n", + "Returned IDs: 993324,991258,998272,994349,996600,993079,994765,994587,996810,995130,992223,993347,992178,996175,996777,995401,994092,994235,993548,998543,994743,995017,991255,996314,994119,996930,998950,997993,990930,990668\n", + "Ground Truth: 993324,991258,998272,994349,996600,993079,994765,994587,996810,995130,992223,995375,993347,992178,996175,996277,996777,995401,994092,994235,993548,998543,994743,995017,991255,996314,998355,994119,996930,996318\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 825\n", + "Returned IDs: 994682,998006,997666,995209,990712,990998,992209,995798,993877,997527,998122,998084,992652,992237,990091,996209,992892,998055,990194,993224,996960,999273,990226,994280,997618,995223,991382,998671,990806,997506\n", + "Ground Truth: 994682,998006,997666,995209,990712,990998,992209,995798,993877,997527,998122,998084,992652,992237,995859,990091,996209,992892,998055,990194,993224,996960,999273,990226,994280,997618,995223,991382,999441,998671\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 826\n", + "Returned IDs: 996362,993478,997020,992169,997723,992161,997918,998758,997420,997395,993030,995516,997675,995203,997693,999075,995783,996305,998593,994248,990103,999179,994409,996001,996918,994280,992124,996867,993809,996075\n", + "Ground Truth: 996362,993478,997020,992169,997723,992161,997918,998758,997420,997531,997395,993030,995516,997675,991799,995203,997693,999075,995783,996305,998593,992933,994248,990103,990279,999179,994409,996001,996918,994280\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 827\n", + "Returned IDs: 994051,993396,996899,999601,991399,994801,991052,996042,995963,996918,995342,997723,990189,994280,998123,993805,993727,996315,996127,995592,998267,993014,998778,993165,994817,998397,990103,992719,991765,997693\n", + "Ground Truth: 994051,993396,996899,999601,991399,994801,998766,997714,991052,996042,995963,996918,995342,997723,990189,994280,992569,998123,999596,993578,993805,993727,992933,996315,999820,996127,997803,995592,998267,993014\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 828\n", + "Returned IDs: \n", + "Ground Truth: 997116,996495,994142,999273,997506,998456,993139,996328,990712,995571,996408,993993,994680,990408,994196,990293,998168,999985,991667,990091,999139,996431,991633,994012,993560,990705,994296,991552,990479,990449\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 829\n", + "Returned IDs: \n", + "Ground Truth: 990310,993143,998033,991157,996840,999257,991392,995584,999122,999144,991609,991015,991840,995515,990580,999676,996829,997593,992950,992734,997379,998909,993355,997320,996007,994757,999546,992979,997037,997918\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 830\n", + "Returned IDs: 993324,995375,990930,995130,990646,994092,991712,998272,998349,997675,996277,996328,993561,996591,994743,993293,996809,994409,992790,995080,993931,993347,999533,997309,993738,995408,995560,994488,999091,993280\n", + "Ground Truth: 993324,995375,997184,990930,995130,990646,994092,991712,992933,993533,998272,998349,997522,996277,997675,996328,993561,996591,991052,994743,993293,996809,994409,992790,995080,993931,993347,999533,997309,993738\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 831\n", + "Returned IDs: 992809,998593,996001,990255,998103,993232,990314,997374,994467,997332,996657,995158,990364,993819,991212,999099,998006,998878,993931,991610,998536,999555,997009,994667,995167,990288,991453,996809,990167,990573\n", + "Ground Truth: 992809,998593,996001,990255,998103,993232,990314,997374,994467,997332,996657,995158,990364,993819,991062,991212,999099,998006,992966,998878,993931,991610,998536,999555,990786,997009,994667,995167,990288,991453\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 832\n", + "Returned IDs: 990934,997529,996219,996507,999876,990974,996328,999343,995384,991335,996075,995107,996727,998668,999898,992147,990866,998500,999144,995455,995536,999374,997381,998950,990275,992384,992291,997918,997192,993168\n", + "Ground Truth: 990934,995471,997529,996219,996507,999876,990974,996328,999343,995384,991335,996075,995107,996727,991743,990052,998668,999898,995096,992147,990866,999784,998500,999144,995455,995536,999374,991096,993335,997381\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 833\n", + "Returned IDs: \n", + "Ground Truth: 996339,998410,992582,996310,998011,997645,996220,992989,990284,992738,999240,998521,992558,992134,994819,997942,999820,995811,999400,997745,991389,996903,993296,990054,990921,994804,992428,999144,991680,992638\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 834\n", + "Returned IDs: 998002,990167,999273,992862,994640,995835,995780,993231,993560,995231,996104,990551,992752,990449,998536,995257,998861,995695,995168,998716,998959,996209,997506,997965,998787,993866,992721,998623,999555,996406\n", + "Ground Truth: 998002,990167,999273,995326,992862,994640,995835,995780,993231,993560,995231,996104,990551,992752,990449,998536,995257,998861,995695,995168,998716,998959,996209,997506,997965,998787,993866,992721,998623,999555\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 835\n", + "Returned IDs: 997301,997320,995274,998302,990231,995961,997913,999338,997470,998779,990585,990949,991498,991743,995045,993484,990957,990096,999290,992509,999710,999801,994964,999026,993431,998950,993845,990560,997775,998921\n", + "Ground Truth: 997301,997320,995274,998302,990231,995961,997913,999338,999109,997470,998779,990585,990949,991498,991743,995045,993484,990957,999533,990096,999290,992509,999710,994831,999801,999432,994964,999647,999026,993431\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 836\n", + "Returned IDs: 995515,996840,997192,998921,993143,998401,991157,999122,991848,991392,994216,991840,999613,998607,996504,999327,999053,997496,997465,991687,999257,999676,995154,994800,991015,990142,997486,998915,994922,997403\n", + "Ground Truth: 995515,996840,997192,998921,993143,993028,998401,991157,999122,991848,991392,994216,991840,999613,998607,996504,999327,999053,997496,997465,991687,998711,999257,999676,995154,994800,997591,991015,997889,999785\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 837\n", + "Returned IDs: 993544,992646,998465,999144,993941,992183,997605,998951,998170,997571,990647,996658,996961,995824,994757,990509,990478,999883,994778,990994,998011,994100,990934,995336,998210,991511,999092,995793,991379,990275\n", + "Ground Truth: 993544,992646,998465,999144,993941,992183,997605,998951,998170,997571,990647,995014,996658,996961,995824,994757,990509,990478,999883,994778,992040,990994,998011,994100,990934,995336,998210,991511,999092,995793\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 838\n", + "Returned IDs: 991459,990096,997906,998934,999384,999724,992211,999079,998623,997884,993564,990072,993342,999622,994434,998720,995402,991451,993275,999736,994906,997506,998022,994147,996164,997666,996817,993848,997685,994682\n", + "Ground Truth: 991459,990096,997906,998934,999384,992429,999724,992211,999079,998623,997884,993564,990072,993342,999622,994434,998720,993609,995402,991451,998419,993275,999736,994906,997506,998022,994147,996164,997666,996817\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 839\n", + "Returned IDs: \n", + "Ground Truth: 999216,992068,994988,995560,990786,994638,992423,996262,999806,999639,999711,990957,990167,998383,992617,994852,993504,994598,994660,990769,996809,996053,996857,999210,998292,993617,991157,998708,992979,997864\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 841\n", + "Returned IDs: 995443,994046,998097,998726,995080,998623,993143,999784,992636,999122,991999,998950,993512,990096,995806,999676,991801,992944,999210,993917,999701,998180,993260,997309,997301,995515,993866,995097,994640,997550\n", + "Ground Truth: 995443,990648,994046,998097,998726,998550,995080,998623,992933,993143,996292,999784,992636,999122,991999,998950,990257,993512,992318,990096,995806,993719,999676,994835,991801,992944,999210,993917,999701,998180\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 842\n", + "Returned IDs: 999334,993535,993715,996594,996050,998536,995746,999373,991436,995655,999711,992458,997685,996989,990293,992664,997617,995969,995385,994723,991654,990622,993917,992157,995393,996501,998330,993001,994226,999724\n", + "Ground Truth: 990670,999334,993535,993715,998468,996594,996050,998536,995746,999373,991436,995655,999711,992458,997685,996989,990293,992664,992933,997617,995969,995385,994723,991654,992110,990622,993917,992157,990942,997470\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 843\n", + "Returned IDs: 992664,994280,991000,996147,998934,998716,994682,998593,996830,997790,999852,999342,994382,993335,996153,990771,998030,990675,996809,994131,990091,991081,993715,996001,995200,995326,999922,990811,993931,993391\n", + "Ground Truth: 992664,994280,991000,996147,998934,998716,994682,996028,998593,996830,997790,999130,999852,999342,994382,993335,996153,990771,998030,990675,996809,994131,990091,991081,993715,992780,996001,995200,995326,999922\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 844\n", + "Returned IDs: 991470,996760,991257,997506,997019,998536,995168,995257,998281,992110,997265,996594,996395,992721,992793,993194,998331,990292,993715,999373,996245,993866,993936,990942,997666,991610,999129,999994,995231,990194\n", + "Ground Truth: 991470,996760,991257,997506,997019,998536,993381,995168,995257,998281,992110,997265,993076,996594,996395,995998,992721,990976,992793,998468,994374,993194,998331,998214,990292,991945,993715,999373,996245,993936\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 845\n", + "Returned IDs: 991389,993314,997103,995268,998733,998851,999279,999241,994991,993466,997185,991664,990132,991585,993751,990251,999759,994380,997411,995633,999085,992940,993467,998067,990537,993883,993209,993939,996505,992448\n", + "Ground Truth: 991389,993314,997103,995268,998733,998851,999279,999241,994991,999436,993466,997185,991664,990132,991585,992091,993751,990251,999759,994380,998536,997411,996752,995633,999085,992940,993467,998067,990537,993883\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 846\n", + "Returned IDs: 990624,997251,992483,992992,999529,996042,995464,992844,999845,996687,997344,998838,991334,997234,991161,993857,997918,994928,997601,990686,993948,993313,995763,995537,992639,993335,999392,997746,997840,994139\n", + "Ground Truth: 990624,997251,992483,992992,999529,996042,995464,992844,999845,996687,997344,999244,998838,991334,997234,991161,993857,995550,997918,994928,997601,992487,990686,993948,993313,990487,995763,995537,996220,992639\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 847\n", + "Returned IDs: 999682,991374,992740,993530,992493,997232,990733,995807,995105,997574,992677,997675,997515,993263,997634,995521,995625,998806,999961,998080,996695,997149,990243,996094,994774,993149,994677,997723,993160,994301\n", + "Ground Truth: 999682,991374,992740,993530,992493,997232,990733,995807,995105,997574,992677,997675,997515,993263,996579,997634,995521,994823,995625,998806,999961,998080,996695,997149,990243,999327,996094,994774,991061,991052\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 848\n", + "Returned IDs: 998960,992511,999374,991444,996959,998211,990469,992147,998644,999263,992416,996218,997704,998500,993250,992854,997192,998921,992546,994850,990760,997543,997301,993457,999968,993364,994673,994939,990515,990215\n", + "Ground Truth: 998960,992511,999374,991444,996959,998211,990469,992147,998644,999263,992416,996218,997704,991362,998500,993250,992854,997192,998921,992546,994850,990760,997543,993457,997301,999968,993364,994673,994939,990515\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 849\n", + "Returned IDs: 990266,993656,992161,992147,990483,992169,994761,994922,992994,997429,996494,991464,990328,997852,998500,999122,999374,992979,990957,990080,996609,998458,992637,996438,991984,999453,996093,990678,995279,996871\n", + "Ground Truth: 990266,991720,993656,992161,992147,997476,991377,990483,992169,994761,997053,994922,992994,997429,996494,991464,990328,999343,997852,992226,996721,990441,998500,995210,999122,999374,992979,990957,990080,992068\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 850\n", + "Returned IDs: 993319,999798,994465,995501,995476,992372,999886,992595,997331,991667,995768,992868,993001,997565,998006,995073,991779,996809,999430,994356,994682,996580,990194,999256,999425,990173,996328,998536,997340,992832\n", + "Ground Truth: 993319,999798,994465,995501,995476,992372,999886,992595,997331,992752,991667,995768,992868,993001,997565,998006,995073,991779,996809,992110,999430,994356,994682,996580,990194,999256,999425,990173,996328,998536\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 851\n", + "Returned IDs: 996277,992115,997168,997522,994488,990364,994574,992576,990956,999091,998102,993931,998045,992182,991977,991011,995047,993561,991712,998536,992521,992052,995184,991747,999543,995589,991459,998623,994770,996001\n", + "Ground Truth: 996277,992115,997168,997522,994353,994488,990364,994574,992576,990956,995408,999091,998102,993931,998045,992182,991977,991011,995047,993561,995326,991712,998536,992521,992052,995184,993778,991747,999543,995589\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 852\n", + "Returned IDs: 992617,992692,999806,992228,997320,995820,991137,992330,998255,991015,992927,992521,991157,996903,997605,994954,996592,999109,992124,999639,999711,996262,998075,999731,999144,994757,994577,993845,995372,995640\n", + "Ground Truth: 992617,992692,999806,992228,997320,995820,991137,992330,998255,991015,992927,992521,991157,996903,997605,995536,994954,996592,999109,992124,999639,999711,992277,995374,996262,998075,999731,999144,994757,994577\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 853\n", + "Returned IDs: 996695,990319,996813,992801,997742,997075,999646,995493,995521,996607,995146,996472,997077,999475,995450,996762,997675,992952,995423,992605,994554,995702,993385,993899,994835,995891,991699,998910,997975,996112\n", + "Ground Truth: 996695,990319,996813,992801,997742,997075,999646,995493,995521,996607,995146,996472,994823,997077,999475,995450,996762,997675,992952,995423,992605,994554,995702,993385,993899,994835,995891,991699,998910,997975\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 854\n", + "Returned IDs: 997383,999915,996001,998097,992395,995873,996685,995476,993053,994742,992792,995073,997320,990930,999099,992568,996809,999560,998779,999399,997814,992318,991007,995216,999453,998725,991245,993342,996328,994793\n", + "Ground Truth: 997383,999915,996001,991175,998097,992395,995873,996685,995476,993053,994742,992792,995073,997320,990930,999099,992568,996809,999560,998779,999399,997814,992318,991007,996043,995216,999453,998725,991245,993342\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 855\n", + "Returned IDs: 994080,994760,992558,991977,992170,994119,992718,990776,990487,994254,996633,996809,992052,995503,995159,998243,990908,994334,998331,990345,998372,999179,996328,996391,993839,998370,990998,992615,993500,997309\n", + "Ground Truth: 994080,994760,992558,991977,992170,994119,992718,990776,990487,994254,996633,996809,995179,992052,998686,998480,995503,994956,995159,998243,995491,990908,994334,998331,990345,998372,999179,996328,996391,993839\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 856\n", + "Returned IDs: 993289,993845,993637,990438,993301,999338,992825,993719,994081,998021,995401,990855,992176,993715,998726,998536,996918,997002,990289,997918,990096,998950,999724,996328,999791,999516,995932,997970,999453,992318\n", + "Ground Truth: 993289,992933,993845,993637,990438,993301,999338,992825,993719,994081,998021,995401,990855,999698,992176,993715,998726,998536,996918,995080,997002,990289,997918,990096,998950,991758,999533,999724,996328,999791\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 857\n", + "Returned IDs: 990275,996053,998713,995260,996554,991018,990231,992363,997371,990366,992228,991479,999243,995755,992931,997913,992453,993387,997508,999938,992087,994266,992942,999144,998090,992370,994100,991379,997880,997192\n", + "Ground Truth: 990275,997439,996053,998713,995260,996554,994618,991018,990231,992363,990650,997371,990366,997451,992228,991479,997432,999243,995755,993048,996219,995210,998744,992931,997913,992453,993387,997508,999938,992087\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 858\n", + "Returned IDs: 995679,994140,992448,994409,990081,999099,993813,997868,991585,990751,995702,991790,996042,998633,993561,992858,997832,997289,995216,991125,996169,993831,997822,996719,990576,996368,996038,998941,994419,997046\n", + "Ground Truth: 995679,994140,992448,994409,990081,999099,993813,997868,991585,990751,995702,991790,991603,996042,998633,993412,991024,993561,992858,997832,997289,995216,991125,996169,993831,997822,996719,990576,996368,996038\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 859\n", + "Returned IDs: 991618,998075,996987,997998,992448,994954,991117,998403,999421,996187,991357,998979,996621,992521,997176,998255,998910,997478,999121,999800,994106,998162,995813,991253,996169,990341,997909,992763,993672,999731\n", + "Ground Truth: 991618,998075,996987,997998,992448,994954,991117,998403,994773,999421,996187,991357,998979,996621,992521,997176,998255,998910,997478,999121,999800,994106,998162,995813,991253,996169,990341,997909,992763,993738\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 860\n", + "Returned IDs: 990275,993853,991260,992384,992452,999647,997976,992969,998899,994992,991893,993168,998514,990925,996727,997571,996714,990957,992994,999089,995034,992911,990505,995279,995755,995336,992546,994917,998951,995793\n", + "Ground Truth: 990275,996336,993853,991260,992384,992452,999647,995014,997976,994776,991829,992969,998899,994992,994192,990724,991893,990862,990893,993168,998514,990925,997022,996727,997571,992909,996714,990356,990957,992994\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 861\n", + "Returned IDs: 998302,994382,997937,991790,995204,990560,997320,997301,992862,995981,998294,994964,995949,997913,998779,990252,998807,996391,998495,995895,997969,995755,990096,995976,991696,997822,990224,995886,995274,998776\n", + "Ground Truth: 998302,994382,999755,997937,991790,995204,991923,994707,990560,995672,997320,993412,997301,995702,992862,995981,991995,998294,994964,995949,997913,998779,990252,998807,996391,998495,995895,997969,995755,990096\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 862\n", + "Returned IDs: 996219,997913,990843,997301,994266,993662,998495,995942,990958,999542,999374,998540,995927,990776,997420,995802,995949,997320,998921,994964,991650,999331,994850,995260,997470,994277,998550,999908,998758,990366\n", + "Ground Truth: 996219,997913,990843,997301,991085,994266,993049,993662,998495,992946,995942,990958,999533,998380,997741,999542,999374,998540,995927,990776,997420,995802,995949,997320,998921,994964,991650,999331,994850,997855\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 863\n", + "Returned IDs: 999912,992337,998115,994972,990366,997453,999544,999876,999949,996052,990179,995107,998122,990585,990089,992047,999791,992907,995716,993139,999711,990043,995818,998349,996001,999898,997796,993484,995372,998849\n", + "Ground Truth: 999912,992337,998115,994972,990366,997453,999544,999876,999949,996052,990179,995107,998122,990585,995222,990089,992047,999791,996947,992907,995716,993139,999711,990043,995818,998349,996001,999898,997301,997796\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 864\n", + "Returned IDs: 992069,998779,998950,997918,996262,995694,994360,990949,993781,993504,993071,994450,993260,997002,995602,990957,997320,993492,999639,997640,991677,998170,993484,996140,994964,995560,996001,996196,999453,994742\n", + "Ground Truth: 992069,998779,998950,997918,996262,995694,994360,990949,994334,993781,993504,993071,994449,994450,993260,997002,995602,990957,997320,990930,993492,997739,999639,997640,991677,998170,993484,995826,996140,992532\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 865\n", + "Returned IDs: 995550,996455,991870,992973,991990,994175,995239,992981,993357,999775,991769,996328,992239,992784,999399,990956,995699,991111,992450,995159,997244,995637,999660,997153,993991,999924,997041,992713,994315,995184\n", + "Ground Truth: 995550,996455,991870,992973,991990,994175,995239,993980,992981,993357,999775,991769,996328,992784,992239,999399,990956,995699,991111,992450,995159,997244,995637,999660,997153,993991,999924,997041,993006,992713\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 866\n", + "Returned IDs: 998558,994679,994098,995691,996882,999435,998993,990796,998652,994193,990765,999171,994605,992056,994080,995618,990852,993718,996428,993267,997864,990487,996413,999158,998855,996930,993976,991377,990998,993500\n", + "Ground Truth: 998558,994679,994098,995691,996882,999435,998993,990796,998652,994193,990765,991541,999171,994605,992056,994080,995618,990852,993718,996428,993267,994689,997864,990487,996413,990133,999158,998855,996930,990189\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 867\n", + "Returned IDs: 999805,996867,998162,994409,997844,993767,997868,994730,990631,995702,996786,992582,997031,997723,995052,994954,993209,995355,998181,993649,999459,991508,998962,993856,994742,990435,996038,997544,992448,999787\n", + "Ground Truth: 999805,996867,998162,994409,997844,998943,993767,993466,997868,990361,994730,990631,995702,994060,995766,996633,996786,992582,997031,997723,995052,994954,993209,992063,995355,998181,993649,999459,991603,991508\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 868\n", + "Returned IDs: 997554,994793,998060,990845,995216,993422,990081,992792,991585,995575,999099,996147,993841,996657,996001,990662,990255,996368,999402,994106,993344,990314,995679,990211,998878,993804,993032,998593,994013,990751\n", + "Ground Truth: 997554,994793,998060,990845,995216,993422,990081,992792,991585,995575,999099,996147,993841,996657,996001,994764,990662,990255,995963,996368,999402,994106,993344,990314,995679,990211,998878,993804,993032,998593\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 869\n", + "Returned IDs: 996411,995482,997374,993466,995020,990435,999627,998156,990293,999599,994142,993209,994730,991439,992024,998221,994802,999179,995274,995633,998067,990758,998913,995289,994817,993747,996867,999833,993725,994541\n", + "Ground Truth: 996411,995482,997374,993466,995020,995002,990435,999627,998156,990293,999599,994142,993209,999785,994730,991439,992024,998221,991480,994802,998586,999179,995274,995633,998067,991075,995401,990758,998913,995374\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 870\n", + "Returned IDs: 997730,994409,998941,996077,993561,998697,997868,997832,996368,994636,992858,992169,991958,997723,993717,996867,991530,993813,998186,995894,995702,999343,993143,996042,990634,995679,998545,995701,998382,993882\n", + "Ground Truth: 997730,994409,998941,996077,993561,998697,997868,997832,996368,997630,994636,992858,992169,991958,997723,993717,996867,991530,993813,998186,995894,992268,995702,999343,993143,996042,994107,990634,990399,995679\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 871\n", + "Returned IDs: 999851,995336,997320,999883,997371,994415,996143,993364,997571,999982,997842,990231,995372,991782,992027,996516,999263,995841,999810,994164,998921,998170,995865,991379,991662,997913,996504,995515,994854,990958\n", + "Ground Truth: 999851,995336,997320,999883,997371,994415,996143,993364,997571,998344,999982,997842,990231,995372,991782,992027,996516,999263,995841,999810,994164,998921,998170,995865,991379,991662,995422,992789,997913,996504\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 872\n", + "Returned IDs: 996362,995783,994313,996087,992997,993497,997746,995184,990872,997997,995464,991285,994280,997244,990021,994196,996911,991111,999759,996276,990103,997713,993256,993478,993621,991981,994541,995696,992526,998201\n", + "Ground Truth: 996362,995783,994313,996087,992997,993497,997746,995184,990872,990149,997997,995464,991285,994280,990343,997244,990021,994196,999893,996911,991111,999759,996276,990103,997713,993256,993478,993621,991981,992784\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 873\n", + "Returned IDs: 998382,996371,993561,999068,995289,997386,995401,995200,990137,996219,996042,992068,993433,999367,995535,994307,994577,996769,997406,991650,991996,998045,998495,995923,996805,990435,990096,992944,999780,998039\n", + "Ground Truth: 998382,996371,993561,999068,995289,997386,995401,995200,990137,996219,991179,996042,992068,993433,995535,999367,994307,994577,996769,997406,991650,991996,998045,998495,999598,995923,991921,996805,990435,990096\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 874\n", + "Returned IDs: 991255,997914,997542,993738,997918,990814,998697,992979,994552,991447,994280,996328,994163,993256,996305,993079,997251,997555,990776,998860,996848,996721,998123,993976,995401,999820,994040,993062,995559,990930\n", + "Ground Truth: 991255,997914,997542,993738,997918,991948,990814,998697,995281,998567,992979,994552,992012,998097,991447,998427,994280,994972,993143,996328,994163,993256,996305,993079,995600,997251,997806,997555,990776,998860\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 875\n", + "Returned IDs: 998586,998790,994537,996395,991610,992136,997009,990912,991667,990167,991571,993995,990194,999216,996209,992721,994643,996809,991513,991332,991975,990991,994451,991608,996960,999555,990786,997145,991139,997328\n", + "Ground Truth: 995922,998586,998790,994537,996395,991624,993381,991610,996424,990429,992136,997009,990912,991667,990167,991571,993995,990194,999216,994981,996209,992721,990450,994643,996809,993194,991513,991332,991975,990991\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 876\n", + "Returned IDs: 998921,995372,990585,997286,997320,998721,996318,991503,995841,999338,999526,997984,990083,997113,998445,991348,991498,995515,993018,994618,995659,995375,997301,999698,998950,996653,998302,991545,996516,997470\n", + "Ground Truth: 998921,995372,990585,997286,997320,998721,996318,991503,995841,999338,999526,997984,990083,997113,998445,991348,991498,995515,993018,994618,995659,995375,997301,999698,998950,996653,999019,998302,991545,996516\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 877\n", + "Returned IDs: 997320,994964,997920,992331,992744,995357,994873,995274,993168,998764,999437,995336,998228,995438,998365,991776,990210,999740,997178,990535,996219,997330,992692,993484,997028,999344,992942,994432,991883,996001\n", + "Ground Truth: 990269,997320,994964,997920,992331,992744,999159,995357,994873,998470,994225,995274,993168,998764,999437,995336,991433,993066,998228,999340,995438,993909,998365,991776,990210,997178,999740,994584,990535,996219\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 878\n", + "Returned IDs: 995955,998242,997215,990125,994305,995503,992306,992824,991333,995804,990538,992170,990721,990192,991034,993140,991834,998260,999908,992490,992113,997849,997547,999430,999639,996742,996510,995724,991977,997055\n", + "Ground Truth: 995955,998242,997215,990125,994305,995503,992306,992824,991333,995804,990538,992170,990721,990192,991034,993140,996629,991834,998260,999908,992490,992113,997849,997547,999430,999639,996742,994155,996510,995724\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 879\n", + "Returned IDs: 997529,993725,994730,998710,994742,991185,995633,994830,999377,992424,993166,990887,998106,993561,993053,995289,998913,999740,997167,994409,996411,993920,996328,993209,994013,994419,996867,992230,996132,995401\n", + "Ground Truth: 997529,993725,994730,998710,994742,991185,995633,994830,999377,992424,993166,990887,998106,993561,993053,995289,998913,999740,990681,997167,994409,996411,993920,996328,992260,993209,994013,994419,996867,992230\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 880\n", + "Returned IDs: 992411,995552,992558,993920,995309,991851,998652,999206,996953,990843,994418,993474,996878,993077,995618,999305,991337,990279,996140,990345,994817,998950,999540,993742,990657,993456,999908,990989,993534,990308\n", + "Ground Truth: 992411,995552,992558,993920,995309,991851,998652,999206,996953,990843,994418,993474,996878,993077,995618,999305,991337,990279,996140,990345,994817,998950,999540,992966,993742,991263,990657,993456,999908,990989\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 882\n", + "Returned IDs: 992032,992293,991076,991203,991954,996028,990662,993561,990811,993725,998476,993857,994835,998823,997529,991958,999286,990097,992636,998197,991301,999922,998593,997554,995701,997822,992605,998105,995216,997079\n", + "Ground Truth: 992032,992293,991076,991203,996225,991954,996028,990662,993561,990811,993725,991087,998476,993857,994835,998823,991603,997529,991958,999286,990399,990097,990033,992636,994677,991771,998197,991301,991466,999922\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 883\n", + "Returned IDs: 997362,990958,996504,993813,997192,990925,996316,999789,998692,995755,999483,998294,990118,997984,997371,995090,997320,999327,993518,991958,995132,995985,995367,998649,995884,997620,991332,998303,996785,992994\n", + "Ground Truth: 997362,990958,996504,993813,997192,990925,996316,999789,998692,995755,992204,999483,998294,990118,997984,997371,995090,997320,999327,993518,991958,995132,995985,991009,995367,998649,997911,997889,993902,993982\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 884\n", + "Returned IDs: 992809,991767,993232,998565,998623,996797,994643,998536,997019,993995,994839,993727,994901,999555,998787,995695,992110,993366,990255,998006,996395,994844,993306,997502,991257,993715,991228,990741,994374,992926\n", + "Ground Truth: 992809,991767,993232,999498,998565,998623,994274,996797,994643,998536,997019,993995,994839,993727,994901,999663,999555,998787,995695,992110,993366,990255,998006,996395,994844,993306,997502,991257,999458,993715\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 885\n", + "Returned IDs: 999869,994449,993260,995261,992748,992041,995336,996296,990532,994955,998951,994100,997498,997113,999698,993512,998550,997320,993529,990772,997918,996592,996075,991951,994171,992880,998090,990949,998246,996210\n", + "Ground Truth: 990778,999869,994449,993260,995261,992748,992041,995336,996296,990532,992933,994955,998951,994100,997498,999019,999351,997113,990519,999698,993512,998550,997320,993529,990772,997918,996592,996075,991951,994171\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 886\n", + "Returned IDs: 994463,992703,991734,994444,994424,991685,995126,995678,995945,995421,990520,990899,990255,992322,996062,998593,999491,991857,998250,990382,991789,994510,998039,993931,997453,996001,993289,992182,991093,993715\n", + "Ground Truth: 994463,992703,991734,994444,994424,991685,995126,995678,995945,995421,990520,990899,990255,992322,996062,998593,999491,991857,990133,998250,990382,991789,994510,998039,993931,997453,996001,993289,992182,991093\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 887\n", + "Returned IDs: 991432,995955,995562,999639,997775,992435,996065,998260,990721,991157,994305,998142,994163,997739,995618,995983,996510,992824,997614,990413,996884,990538,997420,991817,992095,991923,990192,991015,997055,992306\n", + "Ground Truth: 995156,991432,995955,995562,999639,997032,997775,992435,996065,998260,990721,991157,994305,998142,996511,990210,994163,996592,997739,992497,995618,992350,995983,996283,991801,996510,992824,997614,990413,996884\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 888\n", + "Returned IDs: 996759,999977,997002,996810,999587,995401,999595,991666,997554,998593,998779,998530,998123,992790,992884,997910,994819,999026,993032,994474,997012,991968,996657,990399,996088,996001,993289,996864,994946,994409\n", + "Ground Truth: 990304,996759,999977,997002,996810,992637,999587,995401,999595,991666,997554,998593,994178,998779,998530,998123,992790,992884,997910,994819,999026,993032,994474,997012,991968,996657,993820,990399,996088,995374\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 889\n", + "Returned IDs: 995159,990355,998507,994929,996211,994541,996413,997746,998755,997722,996344,996276,992489,990931,994094,997290,992445,994336,992313,994913,996430,993621,997814,998174,992052,997420,992832,999179,992981,992934\n", + "Ground Truth: 995159,990355,998507,994929,996211,994541,996413,997746,998755,997722,996344,996276,992489,990931,994094,997879,997290,992445,994336,992313,999974,994913,996430,993621,997814,998174,992052,997420,992832,999179\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 890\n", + "Returned IDs: 999547,997723,992796,996867,994730,996633,990631,993209,995702,997868,998162,994380,995958,998067,995907,999805,999402,993466,994409,996042,992448,997411,991301,992959,991664,992051,992599,999241,994945,996109\n", + "Ground Truth: 999547,997723,992796,996867,994730,996633,990631,993209,995702,995470,997868,998162,994380,995958,998067,995907,999805,999402,993466,994409,992091,996042,991943,992448,997411,992024,991301,992959,991664,992051\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 891\n", + "Returned IDs: 990957,993283,992668,995752,991677,999561,996894,994852,999806,995974,998600,995580,990949,995562,999714,995443,991801,992309,993781,998495,999898,993222,994979,995519,997903,991923,992124,992423,997864,990383\n", + "Ground Truth: 990957,993283,992668,995752,991677,999561,996894,994852,999806,995974,998600,995580,998232,998455,990949,995562,999714,995443,991801,992309,993781,998495,999898,993222,997032,994979,995519,997903,991923,992124\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 892\n", + "Returned IDs: 994742,999377,990435,994730,990096,993053,999279,998495,999402,995633,991185,990930,993642,998980,993623,992680,994759,992024,995289,990758,996001,993920,993725,999302,994830,990012,993209,996809,995435,999009\n", + "Ground Truth: 994742,999377,990435,994730,990096,993053,999279,998495,999402,995633,991185,990930,993642,998980,993623,992680,994759,998188,995198,992024,995289,990758,996001,997723,993920,993725,999302,994830,990012,995873\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 893\n", + "Returned IDs: 992748,995203,994081,997320,990083,996075,993809,999453,990949,991984,991002,992880,998697,998741,998951,995596,995250,993781,990532,995016,999317,998090,999639,990697,999698,991609,998232,999701,990339,999076\n", + "Ground Truth: 992748,995203,994081,997320,992497,990083,996075,993809,990778,999453,990949,991984,991002,992880,998697,998741,999338,998951,995596,995250,993781,995398,990532,995016,999317,998090,999639,990697,999698,991609\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 894\n", + "Returned IDs: 994336,997309,993877,994280,993775,995861,996328,991378,994119,994956,991334,992147,993335,998123,997822,998950,996855,995438,999533,995125,993929,996050,991007,997918,994964,992052,996521,998495,997002,998779\n", + "Ground Truth: 994336,997309,993877,994280,993775,992933,995861,996328,991378,994119,994956,999338,991334,991720,998076,992147,990998,993335,998123,997822,997159,998950,996855,995438,999533,992789,995125,998369,993929,996050\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 895\n", + "Returned IDs: 995964,991430,997495,998417,991923,994126,997864,994543,995138,996163,990776,997407,999716,990894,999826,995510,994165,993572,993828,998142,996882,994163,994904,995580,994213,990908,990930,999171,995986,998061\n", + "Ground Truth: 995964,991430,997495,998417,991923,994126,997864,994543,995138,996163,990776,997407,999716,990894,999826,995510,994165,993572,993828,998142,992280,996882,994163,994904,995580,994213,990908,996216,990930,998904\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 896\n", + "Returned IDs: 991924,996328,998279,996362,991111,994742,991885,994496,998067,990854,996816,996728,991981,995735,993621,993450,994806,996495,992450,998251,992713,998256,998671,991428,994428,996189,991769,999852,995831,999812\n", + "Ground Truth: 991924,996328,998279,996362,991111,994742,991885,994496,998067,990854,996816,996728,995735,991981,993621,993450,994806,996495,992450,998251,992713,998256,998671,991428,994428,996189,991769,995315,999852,995831\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 897\n", + "Returned IDs: 997475,997652,991153,991075,997480,994891,993621,993611,990696,999496,990600,992480,994765,993055,994558,995600,997714,999205,998405,993165,992449,990776,996429,994307,994280,991723,998072,990097,992327,993004\n", + "Ground Truth: 997475,997652,991153,991075,997480,994891,993475,993621,993611,990696,999496,990600,992480,994765,991426,993055,995145,994558,995600,994313,997714,999205,998405,993165,992449,996856,990776,996429,994051,994307\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 898\n", + "Returned IDs: \n", + "Ground Truth: 991593,995260,992432,999374,991999,994922,990278,998734,997620,998899,996351,990545,994497,991352,991889,999676,997985,990958,991172,999810,993885,992203,997591,994776,990760,990026,991406,991720,994266,999658\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 899\n", + "Returned IDs: 997992,993504,995918,991595,997790,999735,996067,999811,992082,996082,990494,992747,993737,995167,993024,991212,994134,993071,990434,992864,993611,991741,996196,996599,999093,991425,998732,990196,992247,994537\n", + "Ground Truth: 997992,993504,995918,991595,997790,999735,996067,999811,996986,992082,993609,996082,990494,992747,993737,995167,993024,991212,992665,994134,993071,990434,992864,993611,991741,994815,996196,996599,994651,999093\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 900\n", + "Returned IDs: 995193,996581,997998,998527,997176,998255,991769,999047,992311,998075,995397,992521,996277,995640,995661,991922,991981,995820,993379,992097,998670,994742,996728,994954,998910,997612,998637,997091,990712,999208\n", + "Ground Truth: 995193,996581,997998,998527,997176,998255,991769,999047,992311,998075,995397,994773,992521,996277,995640,995661,991922,991981,995820,993379,992097,998670,994742,994690,996728,991057,994954,998910,997612,998637\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 901\n", + "Returned IDs: 995986,992718,992258,994163,997689,998694,999359,996163,991637,996926,995301,994165,993463,992645,996669,992558,997864,998079,997692,996214,991430,999432,995436,990367,997558,997538,999826,991977,996620,996127\n", + "Ground Truth: 995986,992718,992258,994163,994072,997689,998694,999908,999359,996163,991637,996926,997395,995301,994165,994195,993463,992645,991172,998758,996669,992558,997864,998079,997692,996214,991430,999432,999385,993609\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 902\n", + "Returned IDs: 992289,995948,995289,995030,995633,996867,995190,998196,993261,997319,997630,991810,996411,997947,999957,993433,999375,994110,990516,998913,995401,990814,992364,996302,998765,994204,992827,994283,994783,995078\n", + "Ground Truth: 992289,995948,995289,995030,995633,996867,995190,998196,993261,997319,998557,995494,997797,997630,991810,996411,997947,999957,991447,993433,999375,994110,990516,998913,995401,990814,992364,996302,998765,994204\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 903\n", + "Returned IDs: \n", + "Ground Truth: 999426,991984,990142,999769,993257,991013,998753,994971,997551,993785,992328,994489,998498,994789,996075,993582,991218,992635,995515,998734,990648,997584,995033,997255,996631,995108,993675,996829,996840,998227\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 904\n", + "Returned IDs: 992827,998779,994382,993744,999788,995300,995200,994595,990722,998543,997529,994682,998765,990934,996219,991590,995401,994742,991885,995861,990814,994783,999138,990701,998862,995633,999144,997918,998055,998671\n", + "Ground Truth: 992827,998779,994382,993744,999788,995300,992748,999916,995200,994595,990722,998543,997529,994682,991334,998765,997348,990934,996219,993856,993412,991590,995401,994742,991885,995374,995861,990814,997933,991986\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 905\n", + "Returned IDs: 995113,994191,991962,993931,992771,996062,998593,999543,996001,994280,997675,995168,997453,990407,996733,995536,992437,999984,990802,998723,997506,998250,994463,999091,995017,999994,996028,996505,995421,994424\n", + "Ground Truth: 995113,994191,991962,993931,992771,996062,998593,999543,995401,996001,994280,997675,992793,995168,997453,990407,996733,996071,995536,999461,992437,990998,999984,990802,998723,997506,998250,994463,997732,999091\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 906\n", + "Returned IDs: 996890,994641,993518,992709,992107,997709,990595,995353,994630,995949,994192,993734,997535,993005,994939,990863,995367,997371,990397,995090,997972,994281,997525,996961,994757,995910,998378,996785,992511,997880\n", + "Ground Truth: 996890,994641,993518,992709,992107,997709,990595,995353,994630,995949,994192,993734,997535,993005,999112,994939,990863,992004,995367,997371,990397,995090,997972,994281,997525,996961,994757,995910,994953,998378\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 907\n", + "Returned IDs: \n", + "Ground Truth: 995802,994778,999710,993492,992442,997006,991009,992443,999982,993504,997483,993852,993048,995826,992309,997913,992119,994670,992931,992228,994831,990231,995161,997605,990957,996993,997579,992066,990890,996785\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 908\n", + "Returned IDs: 992020,993570,998551,996570,998492,998936,994922,995791,991806,994777,998983,997789,998734,997474,991405,990334,992138,992186,999453,999972,994438,995004,992146,994318,990043,991198,994089,990652,992944,992360\n", + "Ground Truth: 992020,993570,998551,996570,998492,998936,994922,995791,991806,990155,994777,998983,997789,998734,997474,991405,990334,992138,992186,999892,999453,998232,999972,994438,995004,992146,994318,990043,990849,990490\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 909\n", + "Returned IDs: 998303,998377,997926,995025,991332,999032,992866,999265,995894,992241,993489,994906,993260,998697,991630,991691,991295,996217,999840,995740,995203,995672,991530,993504,996001,998401,999977,996725,992163,997653\n", + "Ground Truth: 998303,998377,997926,995025,991332,999032,992866,999265,995894,992241,993489,994906,993260,998697,991630,991691,991295,996217,999840,995740,995203,995672,991530,993504,997531,996001,998401,999977,996725,992163\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 911\n", + "Returned IDs: 994783,990340,990852,999126,990126,991330,992827,995633,992566,992598,998913,993725,994878,990985,990069,996809,996302,995861,995289,999957,997733,994742,998804,994541,991464,999372,990929,991418,999740,996411\n", + "Ground Truth: 994783,990340,990852,999126,990126,991330,992827,995633,992566,992598,998913,993725,994878,990985,990069,996809,996302,995861,995289,999957,997733,994742,994317,998804,994541,991464,994751,999372,990929,991418\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 912\n", + "Returned IDs: 999079,990096,999831,996918,999771,997906,998849,997305,999367,994955,992052,992944,996001,995203,995737,994718,995932,995536,998495,996124,997320,994643,996915,994661,991498,994901,999711,992407,995372,998302\n", + "Ground Truth: 999079,990096,999831,996918,999771,997906,998849,997305,999367,994955,992052,992944,996001,995203,995737,994718,994226,995932,995536,998495,996124,995332,994081,997320,994643,996915,995401,994661,991498,994901\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 913\n", + "Returned IDs: 996868,996001,993174,998934,992072,995701,998593,995216,992610,994419,994610,995823,998186,998710,996147,998349,995855,997897,991958,999590,994280,996075,997687,998382,993280,996368,998039,997320,997586,993293\n", + "Ground Truth: 996868,996001,993174,998934,992072,995701,997920,998593,995216,992610,994419,994610,995823,998186,998710,996147,998349,995855,996401,991000,997897,991958,999590,993738,994280,996075,997687,993559,998382,990557\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 914\n", + "Returned IDs: 991527,992448,991969,998205,997868,991795,990845,996987,996368,998278,999250,991220,996187,997909,992521,995864,993537,996256,997176,993003,999453,994409,995813,991892,990603,994106,994140,993813,997727,998910\n", + "Ground Truth: 991527,992448,991969,998205,997868,991795,990845,999866,996987,996368,998278,997216,999250,991220,996187,997909,992521,995864,993537,996256,997176,993003,993412,990341,999453,994409,995813,991892,990603,994106\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 915\n", + "Returned IDs: 992515,997215,999426,993125,990697,993447,991503,995372,994886,996805,997914,996075,997550,997584,998498,991070,993721,990538,992843,993341,995459,991426,991409,997775,994233,991272,992661,990192,998950,998933\n", + "Ground Truth: 992515,997215,999426,993125,997448,994626,990697,993447,991503,995372,994886,996805,997914,995398,996075,997550,997584,998498,991070,993721,990538,992843,993341,995459,991426,991409,997775,995622,994233,991272\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 916\n", + "Returned IDs: 994673,992833,997192,993648,998458,991585,999328,991361,992284,991743,991534,995702,996721,992994,999263,996532,997880,999374,992147,991213,990432,991958,997913,992384,992922,992087,994383,990958,994919,993402\n", + "Ground Truth: 994673,992833,994160,991900,997192,990724,991782,997826,993648,993412,998458,991585,999328,996111,991361,990760,992284,991743,991534,990462,995702,996721,992994,999263,996532,991602,997880,999374,992147,998215\n", + "Recall: 0.6333\n", + "\n", + "Query ID: 917\n", + "Returned IDs: 994013,994203,995544,992098,996898,998060,993377,991057,999689,997059,998162,994412,991507,999023,995216,993217,996328,992000,994409,994488,996001,999402,998045,997554,996765,995766,997466,996322,997566,997020\n", + "Ground Truth: 994013,994203,995544,992098,996898,998060,993377,991057,999689,997059,998162,994412,991507,999023,995216,993217,996328,992000,994409,994488,996001,999402,998045,997554,991255,996765,995766,997466,996322,997566\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 918\n", + "Returned IDs: 991075,999472,990315,992156,995455,995633,995613,998017,996411,994783,995289,994431,995131,999064,999500,994558,998105,998982,994130,997079,992449,991315,993725,992079,995002,995831,999302,994307,999922,999356\n", + "Ground Truth: 991075,999472,990315,992156,997798,996275,995455,995633,995613,998017,996411,995600,994783,994879,995289,994431,995131,999064,999500,994558,998105,998982,994130,999949,997079,992449,991315,993725,992079,995002\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 919\n", + "Returned IDs: 994342,992780,998631,994577,998109,995776,992070,998568,994984,999806,997331,999780,996219,994558,999430,998495,997909,995895,997480,992628,992423,992655,997529,993053,996216,999294,993617,992617,990435,995476\n", + "Ground Truth: 994342,992780,998631,994577,998109,996475,995776,992068,992070,990271,998568,994984,999806,997331,999780,996219,993055,997990,994558,999430,998495,997909,995895,997116,997554,997480,992628,992423,993678,992655\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 920\n", + "Returned IDs: 993727,990391,998175,997693,997942,992809,998507,994407,991052,993980,998537,999773,992295,996098,993453,990297,996918,993155,995048,991054,998481,990479,994051,995790,994280,999105,997806,998267,994187,996395\n", + "Ground Truth: 993727,990391,998175,997693,998430,997942,992809,998507,994407,991052,993980,998537,999773,992295,996098,993453,990297,996918,993155,995048,991054,998481,990479,994051,995790,994280,999105,997429,997806,998267\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 921\n", + "Returned IDs: 997261,994389,993357,990610,996591,994435,993079,997168,998049,996245,999438,997393,991850,993280,993738,993715,990810,993931,992239,994040,991551,997494,990264,993293,992832,998891,995059,996277,996505,994076\n", + "Ground Truth: 997261,994389,993357,991904,990610,996591,994435,995361,993079,997168,998049,996245,999438,997393,991850,993280,993738,990769,995359,993715,990810,993931,992239,994040,991551,997494,990264,993293,992832,998891\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 922\n", + "Returned IDs: 996840,996686,995495,998582,998523,995515,997486,991392,993091,998244,999524,994195,990026,998692,995004,991485,999453,993143,991462,990300,992309,999613,998996,996351,992328,998837,991149,995588,991889,994922\n", + "Ground Truth: 996840,996686,995495,998582,998523,995515,993028,997486,991392,993091,998244,999524,994195,990026,998692,995004,991485,999453,993143,991462,990300,992309,999613,998996,996351,995401,992328,990702,998837,991149\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 923\n", + "Returned IDs: 995289,996610,997787,995182,997406,991603,993561,996769,997630,996388,995497,992125,998165,999080,994783,997822,997386,996042,991954,997058,996080,994742,995200,996473,991923,994396,993725,991958,993844,990984\n", + "Ground Truth: 995289,996610,997787,995182,997406,991603,993561,996769,997630,995497,996388,992125,998165,999080,994783,997822,997386,996042,991954,997058,996080,994742,995200,998999,996473,991923,994396,993725,991958,993412\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 924\n", + "Returned IDs: 998996,999453,996485,990965,995850,996419,998001,990043,995515,991485,991407,998837,997486,993237,992835,991131,994439,990827,997219,990858,998244,991157,993282,994800,996840,992944,998937,998782,994216,995372\n", + "Ground Truth: 998996,999453,996485,990965,995850,996419,998001,990043,995515,991485,991407,998837,997486,993237,992835,991131,994439,990827,997219,998236,996652,990858,998244,991157,993282,994800,998116,996840,992944,999671\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 926\n", + "Returned IDs: 994830,996132,994204,990340,990711,994742,994783,995633,998913,990012,998710,997319,993053,998196,996411,991330,999377,995289,994529,992289,996302,993433,996049,995401,997529,995200,996809,990435,993166,997797\n", + "Ground Truth: 994830,996132,994204,990340,990711,994742,994783,995633,998913,990012,998710,997319,993053,998196,996411,991330,999377,995289,994529,992289,996302,993433,996049,995401,997529,995948,995200,996809,990435,993166\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 927\n", + "Returned IDs: 997513,991667,998335,992778,991610,990330,997145,990177,994467,994996,995655,992517,992561,998781,992721,990437,997009,991470,990293,992209,996134,996501,990292,990194,998720,997242,997328,991569,993381,997666\n", + "Ground Truth: 997513,991667,998335,992778,991610,990330,997145,990177,994467,992136,994996,995655,992517,992561,998781,992721,990437,997009,991470,991038,990293,992573,992209,996134,996501,990292,990194,996395,998720,997242\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 928\n", + "Returned IDs: 991037,991212,991227,993855,999510,991956,994221,996213,992315,997897,993504,995562,995213,998250,994510,992819,992168,990596,995500,996884,995167,992176,994302,991648,991425,994743,999139,990409,994101,991245\n", + "Ground Truth: 991037,991212,991227,993855,999510,991956,994221,996213,992315,997897,993504,995562,995213,998250,994510,992819,992168,996677,994468,990596,995500,991596,996884,995167,992176,994302,991648,991425,995421,993981\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 929\n", + "Returned IDs: 993209,995289,995633,996815,994529,995766,995401,997262,998913,991507,991958,994742,995268,995778,992488,993831,998165,998476,991954,993997,996302,999957,990126,993219,996867,994783,998196,996388,994013,993586\n", + "Ground Truth: 993209,995289,995198,995633,996815,996930,994529,995766,995401,997262,998913,997797,991507,991958,994742,999587,995268,996411,995778,990157,992488,993831,998165,998476,991954,993997,996302,999957,990126,993219\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 930\n", + "Returned IDs: 993426,998921,990898,996516,997984,997320,998445,995515,998721,996135,994415,997292,997192,997301,994757,993253,998294,993419,992451,997470,993484,995306,997571,997291,998246,991720,995841,999327,995372,992942\n", + "Ground Truth: 993426,998921,990898,996516,997984,997320,997873,998445,995515,998721,996135,994415,999036,997292,997192,997301,994757,993253,995401,998294,993419,996560,992451,997470,993484,998741,995306,997571,997291,998246\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 931\n", + "Returned IDs: 995299,990074,998803,993413,992456,996183,998949,997340,996986,990730,996408,992372,998903,993335,994682,990998,998716,990700,991919,997577,995393,996357,992628,995313,990173,992664,994269,993917,993101,991382\n", + "Ground Truth: 995299,990074,998803,993413,992456,996183,998949,997340,996986,990730,996408,992372,998903,993335,994682,990998,998716,990700,991919,997577,995393,996357,992628,995313,990173,992664,994269,993917,997116,993101\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 932\n", + "Returned IDs: 997918,993289,997002,995826,993504,993492,993512,996511,992571,993781,994959,999840,997790,994268,990096,998090,996062,999019,990399,997320,997977,999355,991648,994280,997586,991206,996592,998401,994661,995672\n", + "Ground Truth: 997918,993289,997002,995826,993504,993492,993512,999900,996511,994449,992571,993781,992029,994959,999840,997790,994268,990096,998090,996062,999019,990399,997320,997977,990930,999355,991648,994280,997586,991206\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 933\n", + "Returned IDs: 999784,996001,996028,998302,997371,997002,997320,998921,999806,994266,995560,993820,995596,997913,994964,993342,996840,993902,997301,990532,993906,995886,993260,994689,992330,997506,994360,990771,999453,996075\n", + "Ground Truth: 999784,996001,996028,998302,997371,997002,997320,998921,999806,994266,995560,993820,995596,998720,997913,993834,994964,993342,996840,993902,997301,990532,995375,993906,996096,995886,993260,994689,995852,992330\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 934\n", + "Returned IDs: 999867,994964,995435,999302,996645,992546,990957,997371,995942,997913,998163,997320,995949,999374,994783,993853,995755,993725,990925,993484,998500,994831,995928,998294,993885,994179,999740,990736,999356,998326\n", + "Ground Truth: 999867,994964,995435,999302,996645,992546,990957,997371,995942,997913,998163,997320,995949,999374,994783,993853,995755,998365,990504,993725,990925,993484,998500,992107,994831,995928,993412,998294,999444,993885\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 935\n", + "Returned IDs: 995981,996609,990444,996648,991721,994069,999374,991593,990990,998302,999046,997320,998246,996805,997301,998339,998849,991422,990585,998974,990096,990146,992291,990957,999533,994409,997614,998156,998294,995884\n", + "Ground Truth: 995981,996609,990444,996648,991721,994069,999374,991593,993412,990990,996085,998302,998777,999046,997320,998246,996805,997301,995266,998339,993186,998849,991422,990585,998974,991602,993049,990096,990146,992291\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 936\n", + "Returned IDs: 997700,999581,999054,991507,990537,991722,995106,994861,995268,993209,992448,993313,995778,996607,990372,997738,995820,998162,998910,999279,995518,996505,994380,990924,991424,996813,995401,991699,995521,999047\n", + "Ground Truth: 997700,999581,999054,991507,990537,991722,995106,994861,995268,993209,992448,993313,995778,996607,990372,997738,995820,998162,998910,998283,999279,995518,996505,994380,990924,991424,993046,996813,995401,991699\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 937\n", + "Returned IDs: 992751,995443,996609,992944,997918,991801,996075,999355,995203,997002,998232,995336,994552,996871,993781,996193,999620,990043,995515,998180,996001,993283,996735,997305,995536,992880,991498,991131,994042,995372\n", + "Ground Truth: 992751,995443,996609,992944,997918,991801,996075,999355,995203,997002,998232,995336,994552,996871,993781,996193,999620,994162,990043,995515,998180,996744,996001,993283,994449,995365,996735,997305,995536,992880\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 938\n", + "Returned IDs: 998777,996219,999676,995401,999453,998530,998797,992309,998495,993355,994318,990843,996840,998302,992950,990974,995016,991585,994195,993431,994964,990432,992216,999302,997554,991743,992169,990856,991732,993717\n", + "Ground Truth: 998777,996219,999676,995401,999453,998530,998797,992309,998495,993355,994318,990843,996840,998302,992950,990974,992328,994850,995016,991585,994195,993431,993412,994964,995899,990432,992216,995374,999302,997554\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 940\n", + "Returned IDs: 994897,993342,999806,993617,999430,992628,993528,992807,995393,990173,990194,991846,999555,994155,998787,994150,996809,999834,995895,994122,993076,994269,996184,996944,994834,991148,997906,997084,997378,994682\n", + "Ground Truth: 994897,993342,999806,993617,997335,999430,992628,995039,991271,993528,992807,995393,990173,990194,991846,999555,994155,998214,998787,997126,994150,996809,999834,996660,995895,997196,994122,993076,994269,996184\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 941\n", + "Returned IDs: 993143,997370,990580,994789,995515,999077,993237,991840,999898,998837,999784,998401,991999,998734,992612,996840,994964,993260,990096,997918,992944,998180,991172,990142,996693,999210,999676,994195,997586,998455\n", + "Ground Truth: 993143,997370,990580,994789,995515,999077,993237,991840,999898,998837,999784,998401,991999,997135,998734,992612,996840,994964,993260,990096,997918,992944,995401,998180,991172,990142,996693,999210,999676,994195\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 942\n", + "Returned IDs: 998793,999476,997215,992802,997744,990150,995552,990755,994965,995724,990192,990908,992608,999639,999164,998242,996196,990930,997548,991791,998495,990538,994661,999908,995979,995430,995372,995983,998754,994165\n", + "Ground Truth: 998793,999476,997215,992802,997744,990782,990150,995552,990755,994965,994536,995724,990192,990908,992608,999639,990334,999164,998242,996196,990930,997548,991791,998495,990538,994661,999908,995979,995430,995372\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 943\n", + "Returned IDs: 994879,992582,997728,996673,997353,998225,992502,991431,992293,992862,997304,995981,999761,992481,998708,994304,998986,994894,998697,999243,993809,995927,992183,997154,992852,998200,997913,996714,993970,995701\n", + "Ground Truth: 994879,998347,993222,992582,997728,996673,992760,994265,997353,998225,992502,991431,992293,996948,992862,997304,991680,995981,999761,992481,990862,998708,994304,998986,994894,998697,993072,999243,993809,997620\n", + "Recall: 0.7000\n", + "\n", + "Query ID: 944\n", + "Returned IDs: 992868,995768,991490,997340,999798,991850,991779,992761,999256,991667,995655,990173,992517,995385,995501,994356,995476,993055,990154,990127,995879,993560,992372,995299,992561,998495,991313,997506,996580,993391\n", + "Ground Truth: 992868,995768,991490,997340,999798,991850,991779,992761,999256,991667,995655,990173,992110,992517,995385,995501,994356,999601,995476,993055,990154,995222,990127,991494,995879,993560,998167,992372,995299,992561\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 945\n", + "Returned IDs: 993209,993299,996867,990638,993586,997544,990082,994742,997319,992582,990126,993549,995078,995030,994587,991943,995211,996302,998913,995633,990493,998910,992289,995268,993920,996411,991639,995274,999091,997746\n", + "Ground Truth: 993209,993299,996867,994317,990638,993586,997544,990082,994742,997723,997319,992582,990126,997077,993549,995078,995030,994587,991943,995211,996302,992667,998913,995633,996277,991664,990493,998910,992289,995268\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 946\n", + "Returned IDs: 995547,999530,994409,999367,998545,997723,993717,993561,991923,997191,996042,994948,996769,996038,996918,997406,995401,995507,997832,991459,994002,997675,993143,997289,991052,995923,993271,996048,992130,998045\n", + "Ground Truth: 995547,999530,994409,999367,995994,998545,991061,997723,993717,993561,991923,997191,996042,994948,996769,994026,996038,996918,997406,995401,995507,997832,991459,993122,994002,993143,997675,997289,998250,991052\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 947\n", + "Returned IDs: 991298,992680,992469,995126,993053,993164,994140,993561,990529,990096,990845,996411,994925,996001,996062,996028,999729,990076,994783,999302,991789,995289,995216,998039,995823,994964,999099,997386,990930,994293\n", + "Ground Truth: 991298,992680,992469,995126,998633,993053,992334,993164,994140,993561,990529,990096,990845,996411,994925,996001,996062,996028,999729,990076,994783,999302,991789,995401,995289,997822,998940,995216,993433,998039\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 948\n", + "Returned IDs: 996001,995536,990532,996661,997586,997320,992862,996368,994200,999444,996178,999639,999587,995672,990814,993504,992469,999974,995983,993157,990399,991981,998827,999453,995401,994334,995535,991157,993289,996147\n", + "Ground Truth: 996001,994449,995536,990532,996661,997586,997320,996824,996511,992862,996368,994200,999444,996178,999639,991630,992521,992635,999587,995672,990814,993504,992469,999974,994343,995983,995250,993157,990399,991981\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 949\n", + "Returned IDs: 993209,995268,990361,999581,995106,995289,991630,991664,993920,990593,994742,995766,996867,998748,993586,999047,995778,997411,995052,993751,998188,993053,999653,997915,992521,998045,994409,993997,999279,991507\n", + "Ground Truth: 993209,995268,990361,997723,999581,995198,995106,995289,991630,991664,993920,990593,994742,995766,996867,998748,993586,996277,999047,995778,997411,992293,995052,996042,998999,993751,998188,997612,993053,999653\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 950\n", + "Returned IDs: 995226,993609,992457,997506,999139,996915,990169,990203,992640,995459,992170,995349,998933,996233,992824,991245,994672,995839,994743,993595,996742,997926,997442,992736,995740,992052,991138,993715,993055,992429\n", + "Ground Truth: 995226,993609,992457,997506,999139,996915,990169,990203,992640,995459,992170,995349,998933,996233,992824,991245,994672,995839,994743,993595,993077,996742,997926,997442,992736,995740,992052,991138,995408,993715\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 951\n", + "Returned IDs: 999076,993355,997539,994922,990300,998424,990809,990580,995210,995515,997747,999582,993688,998236,998751,993237,998837,994789,992612,991392,997291,998401,997772,997818,992950,990615,993091,993143,997852,991833\n", + "Ground Truth: 999076,993355,997539,994922,990300,998424,990809,990580,995210,995515,997747,999582,995430,993688,998236,998751,993237,998837,999469,994789,992612,993222,991609,991392,997291,996561,992979,999097,996793,998401\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 952\n", + "Returned IDs: 995358,990075,998568,994897,996217,999963,997617,996429,993866,993260,995203,996764,993055,999079,997965,997906,994832,992082,996075,993696,993342,990697,998697,993504,993560,991517,994680,992617,995455,999653\n", + "Ground Truth: 995358,990075,998568,994897,996217,999963,997617,996429,993866,993260,995203,996764,995622,993055,999079,997965,997906,997577,994832,992082,997506,996075,991630,993696,993342,990697,998697,993504,993560,991517\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 953\n", + "Returned IDs: 997320,990560,997371,998950,996075,994964,997913,991743,990949,997301,991260,998090,996805,993529,990910,998550,992443,995826,990231,999374,997192,993264,998365,995942,996219,992041,998246,995336,993504,992635\n", + "Ground Truth: 997320,990560,997371,998950,996075,994964,997913,991743,990949,997301,991260,998090,996805,993529,990910,998550,992443,992277,995826,990231,999374,997192,993264,998365,995942,996219,992041,998246,995336,993504\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 954\n", + "Returned IDs: 995840,997807,994782,998668,997586,998496,997004,995927,999210,999941,993114,995944,993689,993878,997364,990866,999898,993624,997305,998814,998708,990373,993831,999218,997552,994815,996219,991938,991373,998495\n", + "Ground Truth: 995840,997807,998414,994782,998668,997586,998496,997004,994756,995927,999210,999941,998068,993114,995944,999931,993689,993878,997364,990866,999898,998861,991006,995971,993624,997305,991259,998814,998708,990373\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 955\n", + "Returned IDs: 998921,997913,995693,998302,998950,990432,994964,994318,996053,997320,996219,992277,992944,996915,997470,998170,993431,997371,991999,996075,999210,999374,991743,996178,992978,999343,999077,992309,993143,994919\n", + "Ground Truth: 998921,995826,997913,995693,998302,998950,990432,994964,994318,996053,997320,996219,992277,998294,992944,996915,997470,998170,993431,997371,991999,992328,992140,996075,999210,999374,997379,991743,996178,992978\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 956\n", + "Returned IDs: 992630,999343,991335,992384,996219,990934,992931,994100,990974,990448,996143,999331,998779,997571,998170,997371,997920,996790,999102,990275,999647,991862,996727,998500,994831,994964,995274,990271,998263,999144\n", + "Ground Truth: 992630,999343,991335,992384,996219,990934,990171,992931,996401,994100,990974,990448,996143,999331,998779,997571,998170,997371,997920,996790,999102,998550,997006,990275,999647,996777,991862,990315,996727,998500\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 957\n", + "Returned IDs: 993561,991958,997406,998382,996080,995126,992334,992680,998593,993738,995923,997753,992469,998039,995216,997386,991125,999729,998165,995289,994925,991857,993645,990096,993390,996001,993433,992322,998999,995182\n", + "Ground Truth: 993561,991958,997406,998382,996080,995126,992334,992680,998593,993738,995923,997753,992469,998039,995216,997386,991125,999729,998165,995289,994925,991857,993645,990930,990096,993390,993725,996001,993433,992322\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 958\n", + "Returned IDs: 998827,999122,992558,990620,996140,999806,991009,990538,997913,997124,997859,991157,995596,995737,998892,997227,996404,998950,995983,991272,999562,992423,999804,996918,997320,994577,996196,993492,995203,997305\n", + "Ground Truth: 998827,999122,992558,990620,996140,999806,991009,990538,997913,997124,997859,991157,995596,995737,998892,997227,996404,998950,995983,991272,999562,992423,992971,999804,996918,997320,992612,994577,996196,993492\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 959\n", + "Returned IDs: 993764,996472,991987,994365,990942,990593,997884,991610,995891,996971,990331,995537,992631,995268,994002,998586,992136,991038,997506,992123,999129,997675,996623,998067,993111,992721,993956,990005,994809,994125\n", + "Ground Truth: 993764,996472,991987,994365,990942,993381,990593,997884,999906,991610,995891,996971,990331,995537,992631,995864,990998,995268,994002,991709,993629,998586,992136,991038,997506,992123,999129,997675,998959,992589\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 960\n", + "Returned IDs: 998335,990627,992862,991304,996456,996104,994796,995991,996594,992270,994564,990802,995231,994742,995168,990998,996408,999334,995780,996611,998279,992636,996728,991098,997506,995626,993638,995530,999216,990844\n", + "Ground Truth: 998335,990627,992862,991304,996456,996104,994796,995991,996594,992270,994564,990802,991447,995231,994742,995168,990998,996408,999319,993076,999334,995780,996611,998279,992636,996728,991098,994498,997506,995626\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 961\n", + "Returned IDs: 995935,997221,991463,993433,999742,998045,992125,998906,996125,991367,992556,996994,995439,998520,992521,996769,996719,996112,996323,997359,993844,995052,996277,997675,997406,990984,996048,992115,992089,997046\n", + "Ground Truth: 995935,997221,991463,993433,999742,998045,997187,992125,998906,996125,991367,992556,996994,995439,998520,992521,996769,996719,996112,996323,997359,993844,995052,996277,997675,997406,990984,996048,992115,992089\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 962\n", + "Returned IDs: 995336,997301,995438,994964,995961,994757,998246,999144,994360,999331,999710,997320,990949,995942,993492,992384,997470,990231,998302,997775,999301,994100,998090,990958,996592,995886,993241,992228,994081,990957\n", + "Ground Truth: 995336,997301,995438,994964,995961,994757,998246,999144,994360,999331,993154,999710,997320,990949,995942,993492,992384,997470,990231,998302,997775,999301,994100,998090,990958,996592,995886,993241,992228,994081\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 963\n", + "Returned IDs: 992135,991459,998623,990537,990682,996001,995968,996060,990502,996247,990005,996812,994809,994380,999453,999543,996759,999724,995932,998068,994835,995268,991389,999653,996256,991175,990251,995815,990294,999977\n", + "Ground Truth: 992135,991459,998623,990537,998720,990682,996001,995968,990457,997140,996060,990502,991154,996247,990005,993673,996812,995027,993131,994809,997091,994380,999453,992124,999543,996417,990616,997965,996759,998029\n", + "Recall: 0.5667\n", + "\n", + "Query ID: 964\n", + "Returned IDs: 991168,991464,992547,995283,995295,999519,995861,994742,995456,992334,991789,999126,994817,994853,999302,998224,993725,997933,992408,993831,998779,994765,993717,994783,996028,990266,990392,990889,993561,999341\n", + "Ground Truth: 991168,991464,992547,995283,995295,999519,995861,994742,995456,992334,990928,991789,999126,994817,994853,999302,991334,998224,996906,997933,993725,996308,992408,993831,998779,994765,993717,996147,994783,996028\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 965\n", + "Returned IDs: 994964,997379,995791,993035,992944,994777,999210,998950,996811,993696,993215,992612,990310,998665,994922,996915,991999,999676,990690,993355,992169,993809,996093,994318,993561,995596,997320,993906,996867,999076\n", + "Ground Truth: 994964,997379,995791,993035,992944,994777,996397,999210,998950,996811,993831,993696,993633,994089,993215,992612,990310,998665,995365,994922,996915,991999,999676,990690,993355,992169,993809,996093,994318,993561\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 966\n", + "Returned IDs: 998797,999995,997040,990146,999262,995884,997320,990925,994964,998302,995873,992546,996959,990936,990958,995947,997586,992896,997911,993484,998246,997913,996075,995336,999343,999242,999374,992330,993168,991498\n", + "Ground Truth: 998797,999995,997040,990146,999262,995884,997320,990925,994964,998302,995873,992546,996959,990936,990958,991852,995947,997586,992896,996785,997911,993484,998246,997913,996075,995336,997822,999343,996296,999242\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 967\n", + "Returned IDs: 995349,997685,999079,992664,997673,991148,993866,997906,999443,999576,992209,995969,995231,992275,994661,997369,990096,997666,993965,995064,995895,996733,991477,995695,993445,996124,995735,995393,995168,990622\n", + "Ground Truth: 995349,997685,999079,992664,997673,991148,993866,997906,998468,999443,999576,992209,995969,995231,992275,994661,997369,990096,997666,993965,995064,995895,996733,991477,995695,993445,996124,995735,992752,995393\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 968\n", + "Returned IDs: 992664,994105,998959,995168,991477,990998,994834,995231,994906,998623,997666,996050,998565,992760,993948,997332,999373,990261,998536,990952,992862,995695,997506,995835,993866,996393,995969,998352,995536,990622\n", + "Ground Truth: 992664,994105,998959,995168,991477,990998,994834,995231,994906,998623,997666,996050,998383,998565,991654,992760,993948,997332,999373,990261,998536,990952,999555,992862,995695,997506,995835,991188,993866,996393\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 969\n", + "Returned IDs: 991110,990729,999737,998022,991767,998318,994063,991887,994088,995373,999662,994956,995449,997131,995480,992604,990256,999445,994122,992133,996236,990577,996730,998629,993137,997942,995080,990345,994539,993185\n", + "Ground Truth: 991110,990729,999737,998022,991767,998318,994063,991887,994088,995373,999662,994956,995449,997131,995480,992604,990256,999445,994122,992133,996236,990577,996730,994577,998629,993137,997942,995080,990345,994539\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 970\n", + "Returned IDs: 993484,994964,993648,993642,993238,997320,999374,990399,999453,995755,993389,990224,997737,996785,993143,993885,995961,998294,991743,994922,995886,990958,994673,998692,996854,997796,999827,997822,994266,997301\n", + "Ground Truth: 993484,994964,993648,993642,993238,997320,999374,990995,992047,990399,999453,995755,993389,996959,990936,990224,997737,996785,993143,993885,993412,995961,993663,992334,993400,998294,991743,994922,995886,990958\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 971\n", + "Returned IDs: 995940,991478,992364,997319,997723,998999,995685,991140,991590,995200,995190,996747,990223,991810,999788,997832,995289,993412,992289,992917,990940,995994,990986,998750,993293,991968,994948,995269,996302,993831\n", + "Ground Truth: 995940,991478,992364,997319,997723,998999,995685,991140,991590,995200,995190,996747,990223,991810,999788,997832,995289,993412,992289,992917,990940,995994,990986,998750,993293,991403,991968,991603,994948,995269\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 972\n", + "Returned IDs: 996001,993053,994106,992182,995200,996219,995873,994382,997909,990435,998725,999653,995216,994689,990557,991464,990752,995823,995289,993438,996657,993984,994529,994783,998982,991212,997529,994558,999402,999028\n", + "Ground Truth: 996001,993053,994300,999151,994106,992182,999352,995200,998568,996219,993883,995873,994382,997909,990435,995384,998725,999653,995216,994689,996411,997837,990557,991464,992384,995823,990752,995289,995198,993438\n", + "Recall: 0.6667\n", + "\n", + "Query ID: 973\n", + "Returned IDs: 990296,992161,998221,998649,998495,990957,999898,990399,997192,996869,992107,995942,990435,993656,996001,990934,991335,997582,992087,996219,999144,995289,992124,993168,991096,995755,995536,990096,994961,993648\n", + "Ground Truth: 990296,992161,998221,993412,998649,998495,990957,999898,990399,997920,997192,996869,992107,995942,990435,993656,996001,990934,991335,997582,991311,992087,996219,999144,999710,995289,992124,995899,998039,993168\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 975\n", + "Returned IDs: 995332,995560,991994,996609,995107,998950,997002,994280,998697,994783,999653,995535,994964,990435,995536,996429,992558,996219,991212,991382,992446,993504,999210,993079,994972,994742,999740,997305,996871,991571\n", + "Ground Truth: 995332,995560,991994,996609,995107,998950,997002,994280,991921,998697,994783,999653,996706,995535,994964,995401,990435,995536,996429,992558,996219,991212,991382,992446,993504,999210,993079,994972,994742,995873\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 976\n", + "Returned IDs: 995958,995778,997023,992473,993209,995401,996429,995701,999675,995536,998067,997204,997216,991507,991044,999977,996232,997732,997544,991747,993313,996328,991958,996506,998045,996220,990537,992293,993561,995702\n", + "Ground Truth: 995958,995778,997023,992473,997723,993209,996376,994453,995401,996429,996226,995701,999675,995536,998067,997601,997204,997216,991507,991044,999977,996232,998495,997732,997544,993948,990814,991747,993313,996328\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 977\n", + "Returned IDs: 996379,998186,994589,999977,998045,997058,994946,996368,992197,997554,993561,995401,998779,996391,991630,995268,995873,993412,995701,993275,993390,996028,994409,996169,990525,992367,993813,999587,991415,992362\n", + "Ground Truth: 996379,998186,994589,999977,998045,997058,994946,996368,992197,997554,993561,995401,998779,996391,991630,995268,995873,993412,995701,993275,993390,996028,997723,994409,996169,990525,992367,993813,999587,991415\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 978\n", + "Returned IDs: 993504,993071,992823,997977,995203,992635,992082,999111,991408,991648,999840,991009,990358,991157,991206,997320,997965,992747,994694,997327,999750,992119,998401,993515,990885,992351,990196,998950,994134,992036\n", + "Ground Truth: 993504,993071,992823,997977,995203,992635,992082,999111,992336,991408,991648,999840,991009,990358,992750,991157,991206,997320,997965,992747,994694,997327,999750,992119,998401,993515,990885,992351,990196,991542\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 980\n", + "Returned IDs: 995643,996322,997776,990252,994419,995547,996818,992894,998545,996042,996254,996148,991301,994173,998486,998866,996569,992660,997046,997822,990137,996719,997477,996360,998564,996745,990975,997937,997289,991011\n", + "Ground Truth: 995643,996322,997776,990252,994419,995547,996818,992894,998545,996042,996254,996148,991301,994173,998486,998866,996569,992660,997046,997822,990137,996719,998973,997477,993416,996360,998416,998564,996745,990975\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 981\n", + "Returned IDs: 993857,997309,994119,994742,995166,994564,995295,990328,993384,997914,991334,997452,995456,999635,998180,997822,996689,993180,997173,997374,990392,998779,999843,993256,992703,998224,992582,999533,995633,994199\n", + "Ground Truth: 993857,997309,994119,994742,995166,994564,995295,990328,993909,993384,997208,997914,991334,997452,995456,999635,998180,997822,996689,993180,997173,991447,997374,990392,990133,998779,999843,992473,993256,992703\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 982\n", + "Returned IDs: 997544,990082,992582,990298,996220,997723,994453,993471,996269,998910,999646,991269,996304,998776,991136,995702,991334,995958,995685,999054,991639,999759,991507,993586,996226,990372,997466,990804,993209,990638\n", + "Ground Truth: 997544,990082,992582,990298,996220,998310,997723,994453,993471,991603,996269,998910,999646,991269,996304,998776,991136,995702,990035,991334,996038,995958,995685,999054,991639,999759,996777,991507,993586,996226\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 983\n", + "Returned IDs: 995295,996580,995283,994730,990528,992226,991464,993725,990531,990930,993585,990315,996411,994742,994682,993623,990614,990076,990133,995289,994783,999798,990811,992680,993053,997529,999916,997340,990392,991330\n", + "Ground Truth: 995295,996580,995283,994730,990528,992226,991464,993725,990531,990930,993585,990315,996411,994742,994682,993623,990614,990076,990133,995289,994783,999798,990811,992680,996136,993053,997141,991629,993705,997529\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 984\n", + "Returned IDs: 991471,998086,995701,997576,991459,998861,999555,999994,990194,998212,999384,996623,993561,991630,994884,990167,999663,991038,997906,999977,996649,999724,998022,993275,992384,993342,997154,997242,996406,996809\n", + "Ground Truth: 991471,998086,995701,997576,991459,998861,999555,999994,990194,995319,998212,999384,996623,993561,991630,990708,994884,999653,990167,999663,991038,997906,999977,999647,991154,996649,999958,999724,994906,998022\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 985\n", + "Returned IDs: 990096,999831,995638,993499,998934,992551,991154,998968,999647,990583,998314,999384,990584,999194,998697,996216,999045,998495,995899,996811,996564,997586,993809,999499,990083,994661,993686,993458,997685,993055\n", + "Ground Truth: 990096,999831,995638,993499,998934,992551,993928,997524,991154,998968,999647,990583,998314,999384,990584,999194,998697,996216,999045,998495,995899,996811,996564,997586,993809,999499,992661,999343,990780,990083\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 986\n", + "Returned IDs: 991348,994552,997918,993260,993241,997320,992041,994290,999453,995372,999210,999077,997113,999317,990843,998950,999784,991999,993484,996318,993696,992944,995108,996075,991743,993018,996592,998464,994886,991131\n", + "Ground Truth: 991348,994552,997918,993260,993241,995622,997320,992041,994290,999453,995372,999210,999077,997113,999317,990843,998950,999784,991999,993484,996318,993696,992944,993365,995108,996075,991743,993018,992942,997135\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 987\n", + "Returned IDs: 997204,992169,993288,995289,999587,995359,999099,992685,995685,997918,998530,993412,991810,994409,996328,996220,999820,992979,992239,995401,997395,997002,997630,994906,996903,995994,990940,999977,995611,990223\n", + "Ground Truth: 997204,992169,993288,995289,995374,999587,995359,999099,992685,995685,991151,997918,998530,993909,993412,991810,994409,996328,996220,999820,992979,992239,995401,997395,997002,997630,994906,990337,996903,997447\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 988\n", + "Returned IDs: 998992,997015,993562,990535,991068,993050,994685,999531,991568,994997,994234,992744,993484,994964,991008,997306,994217,995336,990673,995824,996785,997320,993648,999757,991467,992709,996959,997314,993168,994111\n", + "Ground Truth: 998992,997015,993562,990535,991068,993050,991433,994685,999531,991568,994997,994234,992744,993484,994964,991008,997306,994217,995336,990673,995824,996785,997320,993648,999757,991467,992709,996959,997314,993168\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 989\n", + "Returned IDs: 997267,990129,991544,995891,996472,992448,992801,996042,991958,995521,994409,997884,998045,998703,990681,995200,993433,994002,994742,991671,994954,994413,993127,993561,998623,995702,996695,996607,994365,998910\n", + "Ground Truth: 997267,990129,991544,995891,996472,992448,992801,996042,991958,995521,994409,997884,995864,995873,998045,997909,998703,990681,995200,995198,993433,994002,994742,991671,994954,994413,993127,993561,998623,992123\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 990\n", + "Returned IDs: 999139,992457,993567,994280,995600,997480,992824,998109,994130,994196,997442,998405,993055,992480,992237,992736,990929,996016,998251,991382,996270,999985,992052,991723,998631,995823,991128,999205,997050,990705\n", + "Ground Truth: 999139,992457,993567,994280,995600,997480,992824,998109,994130,994196,997442,998405,993055,992480,992237,992736,990929,996016,998251,991382,996270,999985,995459,992052,991723,998631,995823,991128,999205,993621\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 991\n", + "Returned IDs: 996571,991975,997965,994724,991987,993641,991513,991038,999653,997369,995780,996920,997906,991844,997265,992945,998586,994374,998861,995168,992233,996209,990991,995231,992862,996326,994451,998858,990167,995536\n", + "Ground Truth: 996571,991975,996120,993381,997145,997965,994724,991987,993641,991513,991038,999653,997369,995780,996920,997906,991844,997265,993629,992945,998586,994374,998861,990450,995168,992233,996209,990991,995231,992862\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 992\n", + "Returned IDs: 993562,997015,993050,998992,990535,999531,997314,991568,990673,999026,993168,991444,992911,992744,995034,991391,990797,994964,993648,996714,994595,992646,999596,990817,999024,991467,991086,998951,995077,994100\n", + "Ground Truth: 993562,997015,993050,998992,990535,999531,997314,991568,991433,990673,999026,993168,991444,992911,992744,994265,996336,995034,991391,990797,994425,994964,993648,996714,994595,992646,999596,990817,999024,991467\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 993\n", + "Returned IDs: 993018,998950,997320,991157,992423,992617,990843,996262,992521,992944,997370,993840,995560,999432,999806,995596,999453,995515,995552,999210,995203,998255,996140,998232,990083,991801,994746,991348,991224,994577\n", + "Ground Truth: 993018,998950,997320,991157,992423,992617,990843,996262,992521,992944,997370,993840,995560,999432,999806,995596,999453,995515,995552,999210,995203,996368,998255,995250,996140,998232,990083,991801,995899,994746\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 994\n", + "Returned IDs: \n", + "Ground Truth: 994102,990092,991862,991335,996052,999647,990957,990231,996219,992134,994831,990545,997439,997913,996143,992114,995942,991743,995844,999343,999993,994192,993264,997320,990171,999144,993135,992931,996053,994898\n", + "Recall: 0.0000\n", + "\n", + "Query ID: 995\n", + "Returned IDs: 996627,994751,996411,997729,990011,990012,990528,990743,996906,994663,995295,993725,992230,992680,992408,999740,994830,990392,993240,994730,995076,996011,992024,996147,993705,993906,995435,998030,991464,999302\n", + "Ground Truth: 996627,994751,996411,993070,997729,990011,990012,990528,990743,996906,994663,995295,993725,992230,992680,992408,999740,999922,994830,990392,993240,994730,995076,996011,992024,996147,993705,993906,995435,998030\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 996\n", + "Returned IDs: 995193,992521,998527,992046,992657,993021,995820,995640,997023,996581,999047,999394,991922,991102,992917,994508,992311,999731,998075,993107,994185,998255,999992,990295,992595,998910,993998,993379,998060,994954\n", + "Ground Truth: 995193,992521,998527,992046,992657,993021,995820,995640,997023,996581,999047,999394,991922,991102,992917,994508,990099,992311,999731,998075,993107,994185,994690,998255,999992,990295,992595,997554,998910,993998\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 997\n", + "Returned IDs: 991052,997675,996918,993805,996506,991061,991529,993014,996505,995521,996879,998397,999646,991747,993931,998593,994280,995080,993313,990537,996042,992521,995685,992558,998536,995110,999179,992457,992801,999139\n", + "Ground Truth: 991052,997675,996918,993805,996506,991061,997723,995401,991529,993014,996505,995521,996879,998397,999646,991747,993931,998593,994280,995080,993313,990537,994662,993600,998021,996042,992521,995994,994095,992933\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 998\n", + "Returned IDs: 990011,996275,990484,998710,998108,994830,994951,991885,996906,991826,994858,996025,993064,993070,993335,996925,997529,996301,998435,991383,995002,995293,994351,998642,991382,991313,991390,995476,993568,990186\n", + "Ground Truth: 990011,996275,997733,990484,998710,998108,994830,994951,991885,996906,991826,994858,996025,993064,993070,993335,996925,997529,996301,998435,991383,995002,995293,994351,998642,991382,991313,991390,995476,993568\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 999\n", + "Returned IDs: 999162,992654,991977,999638,998334,999387,994010,994672,998118,992718,991540,998142,995503,993018,997407,991432,991157,997320,990843,990926,999954,997816,993976,996262,992124,990487,992944,999019,995019,990532\n", + "Ground Truth: 999162,992654,991977,999638,995560,998334,999387,994010,994672,998118,992718,991540,998142,995503,993018,998567,998280,990930,997407,991432,991157,997320,990843,990926,999954,997816,993976,996262,992124,990487\n", + "Recall: 0.8667\n", + "\n", + "Total query IDs with recall < 1.0: 972\n", + "IDs: [np.int64(0), np.int64(1), np.int64(2), np.int64(3), np.int64(4), np.int64(5), np.int64(6), np.int64(7), np.int64(8), np.int64(9), np.int64(10), np.int64(11), np.int64(12), np.int64(13), np.int64(14), np.int64(15), np.int64(16), np.int64(17), np.int64(18), np.int64(19), np.int64(20), np.int64(21), np.int64(22), np.int64(23), np.int64(24), np.int64(25), np.int64(26), np.int64(27), np.int64(28), np.int64(29), np.int64(30), np.int64(31), np.int64(32), np.int64(33), np.int64(34), np.int64(35), np.int64(36), np.int64(37), np.int64(38), np.int64(39), np.int64(40), np.int64(41), np.int64(42), np.int64(43), np.int64(44), np.int64(45), np.int64(46), np.int64(47), np.int64(48), np.int64(49), np.int64(50), np.int64(51), np.int64(52), np.int64(53), np.int64(54), np.int64(55), np.int64(56), np.int64(57), np.int64(58), np.int64(59), np.int64(60), np.int64(61), np.int64(62), np.int64(63), np.int64(64), np.int64(65), np.int64(66), np.int64(67), np.int64(68), np.int64(69), np.int64(70), np.int64(71), np.int64(72), np.int64(73), np.int64(74), np.int64(75), np.int64(76), np.int64(77), np.int64(78), np.int64(79), np.int64(80), np.int64(81), np.int64(82), np.int64(83), np.int64(84), np.int64(85), np.int64(86), np.int64(87), np.int64(88), np.int64(89), np.int64(90), np.int64(91), np.int64(92), np.int64(93), np.int64(94), np.int64(95), np.int64(96), np.int64(97), np.int64(98), np.int64(99), np.int64(100), np.int64(101), np.int64(102), np.int64(103), np.int64(104), np.int64(105), np.int64(106), np.int64(107), np.int64(108), np.int64(109), np.int64(110), np.int64(111), np.int64(112), np.int64(113), np.int64(114), np.int64(115), np.int64(116), np.int64(117), np.int64(118), np.int64(119), np.int64(120), np.int64(121), np.int64(122), np.int64(123), np.int64(124), np.int64(125), np.int64(126), np.int64(127), np.int64(128), np.int64(129), np.int64(130), np.int64(131), np.int64(132), np.int64(133), np.int64(134), np.int64(135), np.int64(136), np.int64(137), np.int64(138), np.int64(139), np.int64(140), np.int64(141), np.int64(142), np.int64(143), np.int64(144), np.int64(145), np.int64(146), np.int64(147), np.int64(148), np.int64(149), np.int64(150), np.int64(151), np.int64(152), np.int64(153), np.int64(154), np.int64(155), np.int64(156), np.int64(157), np.int64(158), np.int64(159), np.int64(160), np.int64(161), np.int64(162), np.int64(163), np.int64(164), np.int64(165), np.int64(166), np.int64(167), np.int64(168), np.int64(169), np.int64(170), np.int64(171), np.int64(172), np.int64(173), np.int64(174), np.int64(175), np.int64(176), np.int64(177), np.int64(178), np.int64(179), np.int64(180), np.int64(181), np.int64(182), np.int64(183), np.int64(184), np.int64(185), np.int64(186), np.int64(187), np.int64(188), np.int64(189), np.int64(190), np.int64(192), np.int64(193), np.int64(194), np.int64(195), np.int64(196), np.int64(197), np.int64(198), np.int64(199), np.int64(200), np.int64(201), np.int64(202), np.int64(203), np.int64(204), np.int64(205), np.int64(206), np.int64(207), np.int64(208), np.int64(209), np.int64(210), np.int64(211), np.int64(212), np.int64(213), np.int64(214), np.int64(215), np.int64(216), np.int64(218), np.int64(219), np.int64(220), np.int64(221), np.int64(223), np.int64(224), np.int64(225), np.int64(226), np.int64(227), np.int64(228), np.int64(229), np.int64(230), np.int64(231), np.int64(232), np.int64(233), np.int64(234), np.int64(235), np.int64(236), np.int64(237), np.int64(238), np.int64(239), np.int64(240), np.int64(241), np.int64(242), np.int64(243), np.int64(244), np.int64(245), np.int64(246), np.int64(247), np.int64(248), np.int64(249), np.int64(250), np.int64(251), np.int64(252), np.int64(253), np.int64(254), np.int64(255), np.int64(256), np.int64(257), np.int64(258), np.int64(259), np.int64(260), np.int64(261), np.int64(262), np.int64(264), np.int64(266), np.int64(267), np.int64(268), np.int64(269), np.int64(270), np.int64(271), np.int64(272), np.int64(273), np.int64(274), np.int64(275), np.int64(277), np.int64(278), np.int64(279), np.int64(280), np.int64(281), np.int64(282), np.int64(283), np.int64(284), np.int64(285), np.int64(286), np.int64(287), np.int64(288), np.int64(289), np.int64(290), np.int64(291), np.int64(292), np.int64(293), np.int64(294), np.int64(295), np.int64(296), np.int64(297), np.int64(298), np.int64(299), np.int64(300), np.int64(301), np.int64(302), np.int64(303), np.int64(304), np.int64(305), np.int64(306), np.int64(307), np.int64(308), np.int64(309), np.int64(310), np.int64(311), np.int64(312), np.int64(313), np.int64(314), np.int64(315), np.int64(316), np.int64(317), np.int64(318), np.int64(319), np.int64(320), np.int64(321), np.int64(322), np.int64(323), np.int64(324), np.int64(325), np.int64(326), np.int64(327), np.int64(328), np.int64(329), np.int64(330), np.int64(331), np.int64(332), np.int64(333), np.int64(334), np.int64(335), np.int64(336), np.int64(337), np.int64(338), np.int64(339), np.int64(340), np.int64(341), np.int64(342), np.int64(343), np.int64(344), np.int64(345), np.int64(346), np.int64(347), np.int64(348), np.int64(349), np.int64(350), np.int64(351), np.int64(352), np.int64(353), np.int64(354), np.int64(355), np.int64(356), np.int64(357), np.int64(358), np.int64(359), np.int64(360), np.int64(361), np.int64(362), np.int64(363), np.int64(364), np.int64(365), np.int64(366), np.int64(367), np.int64(368), np.int64(369), np.int64(370), np.int64(371), np.int64(372), np.int64(373), np.int64(374), np.int64(375), np.int64(376), np.int64(377), np.int64(378), np.int64(379), np.int64(380), np.int64(381), np.int64(382), np.int64(383), np.int64(384), np.int64(385), np.int64(386), np.int64(387), np.int64(388), np.int64(389), np.int64(390), np.int64(391), np.int64(392), np.int64(393), np.int64(394), np.int64(395), np.int64(396), np.int64(397), np.int64(399), np.int64(400), np.int64(401), np.int64(402), np.int64(403), np.int64(404), np.int64(405), np.int64(406), np.int64(407), np.int64(408), np.int64(409), np.int64(410), np.int64(411), np.int64(412), np.int64(413), np.int64(415), np.int64(416), np.int64(417), np.int64(418), np.int64(419), np.int64(420), np.int64(421), np.int64(422), np.int64(423), np.int64(424), np.int64(425), np.int64(426), np.int64(427), np.int64(428), np.int64(429), np.int64(430), np.int64(431), np.int64(432), np.int64(433), np.int64(434), np.int64(435), np.int64(436), np.int64(437), np.int64(438), np.int64(439), np.int64(440), np.int64(441), np.int64(442), np.int64(443), np.int64(444), np.int64(445), np.int64(446), np.int64(447), np.int64(448), np.int64(449), np.int64(450), np.int64(451), np.int64(452), np.int64(453), np.int64(454), np.int64(455), np.int64(456), np.int64(457), np.int64(458), np.int64(459), np.int64(460), np.int64(461), np.int64(462), np.int64(463), np.int64(464), np.int64(465), np.int64(466), np.int64(467), np.int64(468), np.int64(469), np.int64(470), np.int64(471), np.int64(472), np.int64(473), np.int64(474), np.int64(475), np.int64(476), np.int64(477), np.int64(478), np.int64(479), np.int64(480), np.int64(481), np.int64(482), np.int64(483), np.int64(484), np.int64(485), np.int64(486), np.int64(487), np.int64(488), np.int64(489), np.int64(490), np.int64(491), np.int64(492), np.int64(493), np.int64(494), np.int64(495), np.int64(496), np.int64(497), np.int64(498), np.int64(499), np.int64(500), np.int64(501), np.int64(503), np.int64(504), np.int64(505), np.int64(507), np.int64(508), np.int64(509), np.int64(510), np.int64(511), np.int64(512), np.int64(513), np.int64(514), np.int64(515), np.int64(516), np.int64(517), np.int64(518), np.int64(519), np.int64(520), np.int64(521), np.int64(522), np.int64(523), np.int64(524), np.int64(525), np.int64(526), np.int64(527), np.int64(528), np.int64(529), np.int64(530), np.int64(531), np.int64(532), np.int64(533), np.int64(534), np.int64(535), np.int64(536), np.int64(537), np.int64(538), np.int64(539), np.int64(540), np.int64(541), np.int64(542), np.int64(543), np.int64(544), np.int64(545), np.int64(546), np.int64(547), np.int64(548), np.int64(549), np.int64(550), np.int64(551), np.int64(552), np.int64(553), np.int64(554), np.int64(556), np.int64(557), np.int64(558), np.int64(559), np.int64(560), np.int64(561), np.int64(562), np.int64(563), np.int64(564), np.int64(565), np.int64(566), np.int64(567), np.int64(568), np.int64(569), np.int64(570), np.int64(571), np.int64(572), np.int64(573), np.int64(574), np.int64(575), np.int64(576), np.int64(577), np.int64(578), np.int64(579), np.int64(580), np.int64(581), np.int64(582), np.int64(583), np.int64(584), np.int64(585), np.int64(586), np.int64(587), np.int64(588), np.int64(589), np.int64(590), np.int64(591), np.int64(592), np.int64(593), np.int64(594), np.int64(595), np.int64(596), np.int64(597), np.int64(598), np.int64(599), np.int64(600), np.int64(601), np.int64(603), np.int64(604), np.int64(605), np.int64(606), np.int64(607), np.int64(608), np.int64(609), np.int64(610), np.int64(611), np.int64(612), np.int64(613), np.int64(614), np.int64(615), np.int64(616), np.int64(617), np.int64(618), np.int64(619), np.int64(620), np.int64(621), np.int64(622), np.int64(623), np.int64(624), np.int64(625), np.int64(626), np.int64(627), np.int64(628), np.int64(629), np.int64(630), np.int64(631), np.int64(632), np.int64(633), np.int64(634), np.int64(635), np.int64(636), np.int64(637), np.int64(638), np.int64(639), np.int64(640), np.int64(641), np.int64(642), np.int64(643), np.int64(644), np.int64(645), np.int64(646), np.int64(647), np.int64(648), np.int64(649), np.int64(650), np.int64(651), np.int64(652), np.int64(653), np.int64(654), np.int64(655), np.int64(656), np.int64(657), np.int64(658), np.int64(660), np.int64(661), np.int64(663), np.int64(664), np.int64(665), np.int64(666), np.int64(667), np.int64(668), np.int64(669), np.int64(670), np.int64(671), np.int64(672), np.int64(673), np.int64(674), np.int64(675), np.int64(676), np.int64(678), np.int64(679), np.int64(680), np.int64(681), np.int64(682), np.int64(683), np.int64(684), np.int64(685), np.int64(686), np.int64(687), np.int64(688), np.int64(689), np.int64(690), np.int64(691), np.int64(692), np.int64(693), np.int64(694), np.int64(696), np.int64(697), np.int64(699), np.int64(700), np.int64(701), np.int64(702), np.int64(703), np.int64(704), np.int64(705), np.int64(706), np.int64(707), np.int64(708), np.int64(709), np.int64(710), np.int64(711), np.int64(712), np.int64(713), np.int64(714), np.int64(715), np.int64(717), np.int64(718), np.int64(719), np.int64(720), np.int64(721), np.int64(722), np.int64(723), np.int64(724), np.int64(725), np.int64(726), np.int64(727), np.int64(728), np.int64(729), np.int64(730), np.int64(731), np.int64(732), np.int64(733), np.int64(734), np.int64(735), np.int64(736), np.int64(737), np.int64(738), np.int64(739), np.int64(740), np.int64(741), np.int64(742), np.int64(743), np.int64(744), np.int64(745), np.int64(746), np.int64(747), np.int64(748), np.int64(749), np.int64(750), np.int64(751), np.int64(752), np.int64(753), np.int64(754), np.int64(755), np.int64(756), np.int64(757), np.int64(758), np.int64(759), np.int64(760), np.int64(761), np.int64(762), np.int64(763), np.int64(764), np.int64(765), np.int64(766), np.int64(768), np.int64(769), np.int64(770), np.int64(771), np.int64(772), np.int64(773), np.int64(774), np.int64(775), np.int64(776), np.int64(777), np.int64(778), np.int64(779), np.int64(780), np.int64(781), np.int64(782), np.int64(783), np.int64(784), np.int64(786), np.int64(787), np.int64(788), np.int64(789), np.int64(790), np.int64(791), np.int64(792), np.int64(793), np.int64(794), np.int64(795), np.int64(796), np.int64(797), np.int64(798), np.int64(799), np.int64(800), np.int64(801), np.int64(802), np.int64(803), np.int64(804), np.int64(805), np.int64(806), np.int64(807), np.int64(808), np.int64(809), np.int64(810), np.int64(811), np.int64(812), np.int64(813), np.int64(815), np.int64(816), np.int64(817), np.int64(818), np.int64(819), np.int64(820), np.int64(821), np.int64(822), np.int64(823), np.int64(824), np.int64(825), np.int64(826), np.int64(827), np.int64(828), np.int64(829), np.int64(830), np.int64(831), np.int64(832), np.int64(833), np.int64(834), np.int64(835), np.int64(836), np.int64(837), np.int64(838), np.int64(839), np.int64(841), np.int64(842), np.int64(843), np.int64(844), np.int64(845), np.int64(846), np.int64(847), np.int64(848), np.int64(849), np.int64(850), np.int64(851), np.int64(852), np.int64(853), np.int64(854), np.int64(855), np.int64(856), np.int64(857), np.int64(858), np.int64(859), np.int64(860), np.int64(861), np.int64(862), np.int64(863), np.int64(864), np.int64(865), np.int64(866), np.int64(867), np.int64(868), np.int64(869), np.int64(870), np.int64(871), np.int64(872), np.int64(873), np.int64(874), np.int64(875), np.int64(876), np.int64(877), np.int64(878), np.int64(879), np.int64(880), np.int64(882), np.int64(883), np.int64(884), np.int64(885), np.int64(886), np.int64(887), np.int64(888), np.int64(889), np.int64(890), np.int64(891), np.int64(892), np.int64(893), np.int64(894), np.int64(895), np.int64(896), np.int64(897), np.int64(898), np.int64(899), np.int64(900), np.int64(901), np.int64(902), np.int64(903), np.int64(904), np.int64(905), np.int64(906), np.int64(907), np.int64(908), np.int64(909), np.int64(911), np.int64(912), np.int64(913), np.int64(914), np.int64(915), np.int64(916), np.int64(917), np.int64(918), np.int64(919), np.int64(920), np.int64(921), np.int64(922), np.int64(923), np.int64(924), np.int64(926), np.int64(927), np.int64(928), np.int64(929), np.int64(930), np.int64(931), np.int64(932), np.int64(933), np.int64(934), np.int64(935), np.int64(936), np.int64(937), np.int64(938), np.int64(940), np.int64(941), np.int64(942), np.int64(943), np.int64(944), np.int64(945), np.int64(946), np.int64(947), np.int64(948), np.int64(949), np.int64(950), np.int64(951), np.int64(952), np.int64(953), np.int64(954), np.int64(955), np.int64(956), np.int64(957), np.int64(958), np.int64(959), np.int64(960), np.int64(961), np.int64(962), np.int64(963), np.int64(964), np.int64(965), np.int64(966), np.int64(967), np.int64(968), np.int64(969), np.int64(970), np.int64(971), np.int64(972), np.int64(973), np.int64(975), np.int64(976), np.int64(977), np.int64(978), np.int64(980), np.int64(981), np.int64(982), np.int64(983), np.int64(984), np.int64(985), np.int64(986), np.int64(987), np.int64(988), np.int64(989), np.int64(990), np.int64(991), np.int64(992), np.int64(993), np.int64(994), np.int64(995), np.int64(996), np.int64(997), np.int64(998), np.int64(999)]\n", + "Average Recall across all 1000 queries: 0.7946\n" + ] + } + ], + "source": [ + "#For querying (int filter)- range filter\n", + "import pyarrow.parquet as pq\n", + "import os\n", + "\n", + "#inputs\n", + "int_rate_query = \"99p\" # change to 1p, 50p, 80p, 99p\n", + "top_k = 30\n", + "\n", + "# Mode 1: Single query id\n", + "# Mode 2: Find all query ids with recall < 1\n", + "mode = 2\n", + "query_id = 457 # only used in mode 1\n", + "\n", + "# Map int_rate_query to filter range\n", + "range_map = {\n", + " \"1p\": [10000, 1000000],\n", + " \"50p\": [500000, 1000000],\n", + " \"80p\": [800000, 1000000],\n", + " \"99p\": [990000, 1000000],\n", + "}\n", + "filter_range = range_map[int_rate_query]\n", + "\n", + "# Build full paths\n", + "dataset_path = os.path.join(base_path, dataset_folder)\n", + "test_parquet_path = os.path.join(dataset_path, \"test.parquet\")\n", + "gt_parquet_path = os.path.join(dataset_path, f\"neighbors_int_{int_rate_query}.parquet\")\n", + "\n", + "# Read parquet files\n", + "test_df = pq.ParquetFile(test_parquet_path).read().to_pandas()\n", + "gt_df = pq.ParquetFile(gt_parquet_path).read().to_pandas()\n", + "\n", + "def run_query(query_id):\n", + " query_row = test_df[test_df[\"id\"] == query_id]\n", + " if query_row.empty:\n", + " print(f\"Query ID {query_id} not found in test.parquet\")\n", + " return None\n", + "\n", + " query_vector = query_row[\"emb\"].values[0]\n", + "\n", + " gt_row = gt_df[gt_df[\"id\"] == query_id]\n", + " if gt_row.empty:\n", + " print(f\"Query ID {query_id} not found in ground truth file\")\n", + " return None\n", + "\n", + " ground_truth = gt_row[\"neighbors_id\"].values[0][:top_k]\n", + "\n", + " results = index.query(\n", + " vector=query_vector,\n", + " top_k=top_k,\n", + " filter=[{\"id\": {\"$range\": filter_range}}],\n", + " include_vectors=False,\n", + " )\n", + " returned_ids = [int(r[\"id\"]) for r in results]\n", + "\n", + " intersection = len(set(returned_ids) & set(ground_truth))\n", + " recall = intersection / len(ground_truth)\n", + "\n", + " return returned_ids, ground_truth, recall\n", + "\n", + "\n", + "if mode == 1:\n", + " result = run_query(query_id)\n", + " if result:\n", + " returned_ids, ground_truth, recall = result\n", + " print(f\"Query ID: {query_id}\")\n", + " print(f\"Returned IDs: {','.join(map(str, returned_ids))}\")\n", + " print(f\"Ground Truth: {','.join(map(str, ground_truth))}\")\n", + " print(f\"Recall: {recall:.4f}\")\n", + "\n", + "elif mode == 2:\n", + " print(f\"Finding all query IDs with recall < 1.0 ...\\n\")\n", + " low_recall_ids = []\n", + " all_recalls = []\n", + "\n", + " for qid in test_df[\"id\"].values:\n", + " result = run_query(qid)\n", + " if result:\n", + " returned_ids, ground_truth, recall = result\n", + " all_recalls.append(recall)\n", + " if recall < 1.0:\n", + " low_recall_ids.append(qid)\n", + " print(f\"Query ID: {qid}\")\n", + " print(f\"Returned IDs: {','.join(map(str, returned_ids))}\")\n", + " print(f\"Ground Truth: {','.join(map(str, ground_truth))}\")\n", + " print(f\"Recall: {recall:.4f}\\n\")\n", + "\n", + " print(f\"Total query IDs with recall < 1.0: {len(low_recall_ids)}\")\n", + " print(f\"IDs: {low_recall_ids}\")\n", + " print(f\"Average Recall across all {len(all_recalls)} queries: {sum(all_recalls)/len(all_recalls):.4f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Finding all query IDs with recall < 1.0 ...\n", + "\n", + "Query ID: 1\n", + "Returned IDs: 997420,995382,990103,993165,996814,996219,996878,992878,998104,994051,994199,990678,996062,995438,993727,992193,994119,997997,999441,992780,993088,997331,990284,992731,997602,997098,996811,998691,999191,991031\n", + "Ground Truth: 997420,995382,990103,993165,996814,996219,996878,992878,998104,994051,994199,990678,996062,995438,993727,992193,994119,997997,999441,996830,992780,993088,997331,990284,992731,997602,997098,996811,998691,999191\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 2\n", + "Returned IDs: 996312,991260,995701,992330,996592,993492,997449,997913,992546,997976,990231,990936,990949,995014,994100,996881,992692,992969,993853,993005,996267,994778,991045,994757,992931,990957,999647,992961,990109,993970\n", + "Ground Truth: 996312,991260,995701,992330,996592,993492,997449,997913,992546,997976,990231,990936,993517,990949,995014,994100,996881,992692,992969,993853,993005,996267,994778,991045,994757,992931,990957,999647,992961,990109\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 4\n", + "Returned IDs: 993262,992665,991098,996599,992247,997534,990818,990802,992701,995626,990402,999466,991236,998303,991037,996216,997897,994310,998299,995516,997825,999435,992573,994098,997334,991212,995991,999216,993124,996424\n", + "Ground Truth: 993262,992665,991098,996599,992247,997534,990818,990802,992701,995626,990402,999466,991236,997442,998303,991037,996216,997897,994310,998299,997825,995516,999435,992573,990630,994098,997334,991212,995991,994964\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 5\n", + "Returned IDs: 995274,997320,998779,992994,999144,992630,999026,998184,998365,995755,998951,999090,999647,994100,993412,996959,994964,995701,998862,994757,995702,997976,995336,990949,993648,994730,994894,995438,992331,997737\n", + "Ground Truth: 995274,997320,998779,992994,999144,992630,999026,998184,998365,995755,998951,999090,999647,994100,993412,996959,998633,997925,994964,995701,998862,994757,995702,997976,995336,990949,997377,993648,994730,994894\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 6\n", + "Returned IDs: 992384,995274,993648,995204,993909,991743,992994,998184,993837,999374,993412,998500,995928,999533,992582,994636,990432,992147,995702,994850,998458,996521,992630,992637,998779,995961,991852,997301,996732,996959\n", + "Ground Truth: 992384,995274,993648,995204,993909,991602,991743,993089,992994,998184,991866,993837,999374,997645,993412,990862,998500,995928,999533,992582,994636,990432,992147,995702,994850,998458,996521,992630,992637,991151\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 8\n", + "Returned IDs: 997913,998090,990949,990910,997320,995961,999710,997301,990231,996918,990560,996805,993504,993515,998950,992277,999639,998337,995336,997371,999806,994850,992748,994100,993529,999900,998365,999338,993696,991498\n", + "Ground Truth: 997913,998090,990949,990910,997320,995961,999710,997301,990231,996918,993412,990560,996805,993504,993515,998950,992277,999639,998337,995336,997371,999806,994850,992748,994100,993529,999900,998365,999338,991498\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 10\n", + "Returned IDs: 991067,996273,999530,993530,999193,998316,997057,994677,993223,994301,991190,997723,992545,998650,992493,999961,995963,997675,995723,997098,997824,992582,995685,993455,996046,990930,999646,991052,997083,994948\n", + "Ground Truth: 991067,996273,999530,993530,999193,998316,997057,994677,993223,994301,991190,997723,992545,998650,992493,999961,995963,997675,995723,997098,997824,992582,995685,993455,996046,990930,999646,991052,997083,991840\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 19\n", + "Returned IDs: 990338,997370,993018,991437,997320,996001,996309,999586,998050,991224,996810,993642,990532,990843,993840,994081,991943,991002,991787,998495,996219,990772,994964,992655,994168,999561,992583,990974,990585,990786\n", + "Ground Truth: 990338,997370,995899,993018,991437,997320,996001,996309,999586,998050,991224,996810,993642,990532,990843,993840,994081,991943,991002,991787,998495,996219,990772,994964,992655,991019,994168,995283,999561,992583\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 21\n", + "Returned IDs: 993018,997320,994171,993662,995351,997913,990843,996075,995596,995266,994746,991142,994366,997470,996296,994341,990957,999077,998540,997457,993696,991999,997707,996469,995560,992733,999784,992277,997022,999806\n", + "Ground Truth: 993018,997320,994171,993662,995351,997913,990843,996075,994604,995596,995266,994746,994366,991142,997470,996296,994341,990957,999077,998540,997457,993696,991999,997707,996469,995560,992733,999784,992277,997022\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 22\n", + "Returned IDs: 997290,998557,997517,994110,998543,991334,994893,995030,995159,996809,999924,997947,999788,995948,992566,996747,995190,990814,994422,998913,992289,992468,990722,994824,999942,998194,992588,994541,998340,992364\n", + "Ground Truth: 997290,998557,997517,994110,998543,991334,994893,995030,995159,996809,999924,997947,999788,994062,995948,992566,996747,995190,990814,994422,998913,992289,992468,990722,994824,999942,998194,992588,994541,998340\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 23\n", + "Returned IDs: 996053,990096,997913,997586,994850,995961,995536,998090,996143,995755,995927,998861,994894,999356,999711,993853,997305,994618,998921,992848,997984,998550,994707,991582,994964,998495,992330,997320,992862,993948\n", + "Ground Truth: 996053,990096,997913,997586,994850,995961,995536,998090,996143,995755,995927,998861,994894,999356,999711,993853,997305,994618,998921,992848,997984,992716,998550,994707,991582,995503,994964,998495,992330,997320\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 24\n", + "Returned IDs: 996093,997852,994922,992169,993717,991473,998523,997395,996438,992632,992152,990580,990399,994510,996727,991840,993143,998458,999676,997476,990080,991609,993859,995295,992813,997192,993781,999453,992044,990490\n", + "Ground Truth: 996093,997852,994922,992169,993717,991473,998523,997395,996438,992632,992152,990580,990399,994510,996727,991840,993143,998458,999676,997476,990080,991609,993859,995295,992813,997192,993781,996406,992044,999453\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 26\n", + "Returned IDs: 990487,993500,991977,990998,994254,992558,991447,993335,998686,992052,994148,996068,990684,998406,993975,992170,991620,994006,993583,998536,995408,998480,992718,996262,992933,998623,992280,993965,994080,993715\n", + "Ground Truth: 990487,993500,991977,990998,994254,992558,991447,993335,998686,992052,994148,996068,990684,998406,993975,992170,991620,994006,993583,998536,995408,998480,992718,996262,992933,992554,998623,992280,993965,994080\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 29\n", + "Returned IDs: 998276,993603,994829,997918,997002,995141,992047,996012,993139,996661,993289,991425,997140,990051,996808,997977,992330,993504,992238,994972,996576,994946,998401,992225,994042,995855,997777,992979,990491,999111\n", + "Ground Truth: 998276,993603,994829,997918,997002,995141,996996,992047,996012,993139,996661,993289,991425,997140,990051,996808,997977,992330,993504,992238,994972,996576,994946,998401,992225,994042,995855,997777,992979,990491\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 31\n", + "Returned IDs: 990811,998593,995526,992334,996298,997732,998039,992684,997969,990089,996834,990529,993642,994964,994783,995126,995679,997410,995906,996001,999099,994925,992120,996873,993831,994293,999302,998382,991200,990576\n", + "Ground Truth: 990811,998593,995526,992334,996298,997732,998039,992684,997969,997112,990089,996834,990529,993642,994964,994783,995126,995679,997410,995906,996001,999099,994925,992120,996873,993831,994293,999302,998382,991200\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 34\n", + "Returned IDs: 995130,994659,995332,993808,995750,997273,996128,997542,991961,997412,993347,993079,993304,991990,998860,997442,994871,991343,998891,997199,993931,992567,995107,993499,992358,991061,993324,997555,993548,994410\n", + "Ground Truth: 995130,994659,995332,993808,995750,997273,996128,997542,991961,997412,993347,993079,993304,991990,998860,997442,994452,994871,991343,998891,997199,993931,992567,995107,993499,992358,991061,993324,997555,993548\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 35\n", + "Returned IDs: 994397,996075,995274,991422,999289,996976,998011,995336,997571,993168,995374,990701,993280,999596,997759,990585,997976,996108,994432,991125,990862,996714,997304,994916,997377,994894,996368,999437,993167,994100\n", + "Ground Truth: 994397,996075,995274,991422,999289,996976,998011,995336,997571,993168,995374,990701,993280,999596,997759,990585,997976,996108,994432,991125,990862,996714,997304,994916,997377,994894,996368,999437,993167,997925\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 40\n", + "Returned IDs: 993484,997320,998246,996296,997370,995949,997913,993885,990772,990957,997661,998302,990843,991801,993018,991498,990375,998950,993803,995932,993741,994366,997470,990949,991721,996143,991063,994089,993154,995596\n", + "Ground Truth: 993484,997320,998246,996296,997370,995949,996452,997913,993885,990772,990957,997661,998302,990843,991801,993018,991498,990375,998950,998160,993803,995932,993741,994366,997470,990949,991721,996143,995265,991063\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 41\n", + "Returned IDs: 996395,990194,999555,998006,997205,997242,999994,999576,990167,994467,993106,998122,990360,990230,993381,997149,999435,996209,991148,992771,993122,997086,998066,991767,993224,998429,994796,994638,995848,992517\n", + "Ground Truth: 996395,990194,999555,998006,997205,997242,999994,999576,995167,990167,994467,993106,998122,990360,990230,993381,997149,999435,996209,991148,992771,993122,997086,998066,991767,993224,998429,994796,994638,995848\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 43\n", + "Returned IDs: 994541,999372,999830,995300,998055,998671,994322,998466,994557,995280,996939,992434,993437,995750,995861,993929,993939,998484,995704,991893,996057,990931,994558,991327,994119,993856,995191,993877,994782,998710\n", + "Ground Truth: 994541,999372,999830,995300,998055,998671,994322,998466,994557,995280,996939,992434,993437,995750,995861,993929,993939,998484,995704,991893,996057,990931,991537,994558,991327,994119,993856,995191,993877,994782\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 45\n", + "Returned IDs: 996001,990366,995536,998720,999216,992862,998934,993139,996500,993289,993055,993174,990449,994829,994672,997586,990399,993560,997531,995672,995398,994167,999806,999639,997002,998593,990352,990891,999111,993696\n", + "Ground Truth: 996001,990366,995536,998720,999216,992862,998934,993139,996500,993289,993055,993174,990449,994829,994672,997586,990399,993560,997531,995672,995398,994167,999806,999639,997002,998593,995418,990352,992617,990891\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 46\n", + "Returned IDs: 996504,998921,997889,990917,998170,997192,994775,992147,990647,991743,995515,993518,996586,991127,999374,994939,996218,990515,991782,993250,992087,998721,999327,990275,992789,998401,994919,999263,995415,995621\n", + "Ground Truth: 996504,998921,997889,990917,998170,997192,994775,992147,990647,991743,995515,993518,996586,991127,999374,994939,996218,990515,992379,991782,993250,992087,998721,999327,990275,992789,998401,994919,999263,995415\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 48\n", + "Returned IDs: 994925,995873,993561,995895,991958,998382,998495,992448,993906,998593,993053,992521,992120,999294,996219,998106,993637,997386,996001,992847,998649,994742,997379,994419,995200,996397,995401,998039,990435,997822\n", + "Ground Truth: 994925,995873,993561,995895,991958,998382,998495,992448,993906,998593,993053,992521,992120,999294,996219,998106,993637,994657,997386,996001,992847,998649,994742,997379,994419,995200,995899,996397,995401,998039\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 52\n", + "Returned IDs: 990096,998934,996001,998720,997906,998697,993174,998861,995855,994167,997685,996164,993275,993154,999343,996124,995536,999736,996894,999647,995203,992429,999639,999882,998961,998514,994718,993744,999831,990991\n", + "Ground Truth: 990096,998934,996001,998720,997906,998697,993174,998861,995855,994167,997685,996164,993275,993154,999343,996124,995536,999736,996894,999647,995203,992429,999639,999882,995401,998961,998514,994718,993744,999831\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 53\n", + "Returned IDs: 996436,992912,997640,990797,998779,991750,997355,993412,999144,991651,993526,993725,995755,999740,997256,994990,994916,994436,994894,990535,990388,999026,994964,996714,994838,999021,991173,999344,997192,999596\n", + "Ground Truth: 996436,992912,997640,990797,998779,991750,997355,993412,999144,991651,999587,993526,993725,995755,999740,997256,994990,994916,995923,994436,994894,990535,990388,999026,994964,996714,994838,999021,991173,999344\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 54\n", + "Returned IDs: 997320,999639,993738,990930,990532,995873,997822,996371,995395,996714,992546,990878,993293,992977,992521,991721,995375,999026,990634,996777,994140,991349,990585,992318,991224,995274,995813,993561,996277,999587\n", + "Ground Truth: 997320,999639,993738,990930,990532,995873,997822,996371,995395,996714,992546,990878,993293,992977,992521,991721,995375,999026,999527,990634,996777,990814,994140,991349,990585,992318,991224,995274,995813,993561\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 56\n", + "Returned IDs: 998416,991996,997014,997920,998221,997804,997900,995786,990126,994964,993378,997403,998495,991377,990931,997477,991565,994466,992446,993199,998816,990158,996219,991179,997822,994419,993561,997586,993444,991831\n", + "Ground Truth: 998416,991996,997014,997920,998221,997804,997900,995786,990126,994964,993378,997403,998495,991377,996305,990931,997477,991565,994466,992446,993199,998816,990158,996219,991179,997822,994419,993561,997586,993444\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 58\n", + "Returned IDs: 996196,990159,996690,997224,996523,994384,993670,994372,998286,990303,990824,996655,995503,991550,995290,990989,999860,994950,998411,994964,994912,993158,990538,995599,991458,995410,998697,991791,999716,993809\n", + "Ground Truth: 996196,990159,996690,997224,996523,994384,993670,994372,998286,990303,990824,996655,995503,991550,995290,990989,999860,994950,996744,998411,994964,994912,993158,990538,995599,991458,995410,998697,991791,999716\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 59\n", + "Returned IDs: 996280,991451,995374,991650,999698,995357,993721,994030,993215,997002,998951,993154,991545,995181,991997,992557,999900,994850,998961,992880,994449,990532,997448,997918,994362,996685,990083,994081,993512,998779\n", + "Ground Truth: 996280,991451,995374,991650,999698,995357,993721,994030,993215,997002,994226,998951,993154,991545,995181,991997,992557,999900,994850,998961,992880,994449,990532,997448,997918,994362,996685,990083,994081,993512\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 60\n", + "Returned IDs: 998224,999496,995831,995196,995407,994956,995087,999138,994817,999244,990678,998779,995295,998691,996328,995861,993725,991390,991319,991378,997732,992713,992703,993678,996918,997714,995020,998067,994587,991192\n", + "Ground Truth: 998224,999496,995831,995196,995407,994956,995087,999138,994817,999244,990678,998779,995295,998021,998691,996328,995861,993725,991390,991319,991378,997732,992713,992703,993678,996918,997714,995020,998067,994587\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 61\n", + "Returned IDs: 993738,992651,998831,995701,995184,991836,998697,996534,997059,998060,993280,995079,995873,996828,992609,991530,996657,992293,995640,993948,991175,991007,997897,995216,994488,996001,990005,997554,992052,991576\n", + "Ground Truth: 993738,992651,998831,995701,995184,991836,998697,996534,997059,998060,993280,995079,995873,996828,992609,991530,996657,992293,995640,993948,991175,991007,997897,995216,994488,996001,994163,990005,997554,992052\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 62\n", + "Returned IDs: 991157,999122,991015,998607,997305,991720,992718,997192,993143,998180,996193,998123,995515,994046,992052,997496,993260,998045,998170,996504,992147,998950,991848,995401,993077,998921,995408,995372,998106,993948\n", + "Ground Truth: 991157,999122,991015,997305,998607,991720,992718,997192,993143,998180,996193,998123,995515,994046,992052,997496,993260,998045,998170,996504,992147,998950,991848,995401,993077,998921,995408,995864,995372,998106\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 63\n", + "Returned IDs: 995059,990264,990516,991590,991960,998999,992415,995020,998525,993293,993079,990521,992318,994435,995359,993180,992171,999981,995421,997319,994817,994697,997718,990610,994349,992239,991245,991867,991775,998961\n", + "Ground Truth: 995059,990264,990516,991590,991960,998999,992415,995020,998525,993293,993079,990521,992318,994435,995359,993180,992171,999981,995421,997319,994817,994697,997718,990610,994349,992239,994409,991245,991867,991775\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 65\n", + "Returned IDs: 996809,998695,991571,993617,998790,999009,998786,991890,995461,990288,996370,999711,999430,991139,998849,990361,990538,995375,999194,990489,994451,997821,993515,992452,990930,996830,995560,991532,994643,993696\n", + "Ground Truth: 996809,998695,991571,993617,998790,999009,998786,991890,995461,990288,996370,999711,999430,991139,998849,990361,990538,995375,999194,990489,991052,994451,997821,993515,995401,992452,990930,996830,995560,991532\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 66\n", + "Returned IDs: 999526,997970,991245,999281,991629,997550,994016,997584,996075,993715,992351,992825,995375,995740,992747,993504,995372,998097,997691,999724,993609,992933,994531,996001,995727,992052,994610,992636,990096,999338\n", + "Ground Truth: 999526,997970,991245,999281,991629,997550,994016,997584,996075,993715,992351,992825,995375,995740,992747,993504,995372,998097,997691,999724,993609,992933,994531,996001,995727,994610,992052,992636,990096,992521\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 67\n", + "Returned IDs: 992664,994131,998446,998030,998716,997097,991684,991486,993931,997453,996147,991663,998416,991081,996328,998845,998949,996733,998440,993335,998536,992780,998903,998076,997073,993917,997944,996028,991961,992408\n", + "Ground Truth: 992664,994131,998446,998030,998716,997097,991684,991486,993931,997453,996147,991663,998416,991081,996328,998845,998949,996733,998440,993335,998536,998903,992780,996830,998076,997073,993917,997944,996028,991961\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 69\n", + "Returned IDs: 994100,993492,991673,996805,992330,995336,998495,994334,995833,995968,990231,998921,992944,990271,993512,995357,996001,995274,998090,993154,994088,995536,995826,996282,995401,990648,996219,995260,994449,997605\n", + "Ground Truth: 994100,993492,991673,996805,992330,995336,998495,994334,995833,995968,990231,998921,992944,990271,996769,993512,995357,995748,996001,995274,998090,993154,994088,995536,995826,996282,995162,995401,990648,996219\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 71\n", + "Returned IDs: 995841,995375,995372,992944,999711,996318,990538,990043,994972,998726,991052,995401,990585,997301,998445,996193,999544,991977,991540,997320,995560,996001,998180,995268,995138,996609,993533,999526,999019,995515\n", + "Ground Truth: 995841,995375,995372,992944,999711,996318,990538,990043,994972,998726,991052,995401,990585,997301,998445,996193,999544,991977,991540,997320,995560,996001,998180,995268,998097,995138,996609,992384,993533,999526\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 74\n", + "Returned IDs: 994102,994831,998764,995372,997913,991362,999982,991335,995374,998702,992880,992291,990585,993852,996143,995828,990724,992748,994618,990949,997470,992114,990957,990275,996959,998500,998744,997192,996785,997320\n", + "Ground Truth: 994102,994831,998764,995372,997913,991362,999982,991335,995374,998702,992880,992291,990585,993852,996143,995828,990538,990724,992748,994618,990949,997470,992114,990957,990275,996959,998500,998744,997192,996785\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 75\n", + "Returned IDs: 998981,998287,997320,998246,994225,996296,992902,994889,997178,994450,991155,990020,992880,994214,994249,997470,995116,998540,993484,992748,994542,998634,993529,996075,996085,995700,999132,990742,994964,995400\n", + "Ground Truth: 998981,998287,997320,998246,994225,996296,992902,994889,997178,994450,991155,990020,994214,992880,994249,997470,995116,998540,993484,992748,994542,998634,993529,996075,996085,995700,994416,999132,990742,994964\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 79\n", + "Returned IDs: 992931,997843,992514,990742,996993,991260,998764,997483,995570,990478,992961,993484,991131,997913,990949,997320,998195,999982,992748,999838,999855,998653,992105,992781,991872,992069,995277,991752,995336,997307\n", + "Ground Truth: 992931,997843,992514,990742,996993,991260,998764,997483,995570,990478,992961,993484,991131,997913,990949,997320,996378,998195,999982,992748,999838,999855,998653,992105,992781,991872,992069,995277,991752,995336\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 81\n", + "Returned IDs: 990146,993237,992085,997747,994318,992944,991727,992342,991392,995443,990142,993253,997591,997855,997585,990742,991121,994789,999784,998782,990822,995334,994534,993570,990615,998550,999676,999948,994366,995693\n", + "Ground Truth: 990146,993237,992085,997747,994318,992944,991727,992342,991392,995443,990142,993253,997591,997855,997585,990742,991121,994789,999784,998782,990822,995334,994534,993570,990615,998550,990447,999676,999948,996352\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 84\n", + "Returned IDs: 997913,994964,997371,993237,995515,994778,991801,996727,996811,990142,996840,991727,995928,998359,996001,996438,994277,994415,997593,997291,992140,993143,994971,996592,999290,995886,990580,997320,997370,994360\n", + "Ground Truth: 997913,994964,997371,993237,995515,994778,994475,991801,996727,997350,996811,990142,996840,991727,995928,998359,996001,996438,994277,994415,997593,997291,999426,992140,993143,994971,996592,999290,995886,990580\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 85\n", + "Returned IDs: 995207,991701,996395,998586,991987,996623,992136,995056,999117,995373,994063,990292,994413,998858,998257,994451,998066,991052,994187,999249,994092,999096,997110,998659,992604,993115,997918,999834,994753,993515\n", + "Ground Truth: 995207,991701,996395,998586,991987,996623,992136,995056,999117,995373,994063,990292,994413,998858,998257,994451,998066,991052,994187,999249,994092,995617,999096,997110,998659,992604,993115,997918,999834,994753\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 88\n", + "Returned IDs: 992931,995266,995319,994052,995942,997147,991616,991593,992954,997913,990922,992964,991250,993031,990114,993606,999559,995632,998899,996508,999945,994668,992255,993278,998562,995984,990949,992228,998619,998056\n", + "Ground Truth: 992931,995266,995319,994052,995942,997147,991616,991593,992954,997913,990922,992964,991250,993031,990114,993606,990653,996629,999559,995632,998899,996508,999945,994668,992255,993278,998562,995984,990949,992228\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 89\n", + "Returned IDs: 995755,997437,993484,995244,998649,994894,996630,992296,997320,999740,991999,998246,991991,994814,995435,994359,999599,997304,999708,993696,996053,996436,997305,995841,990949,990786,996075,992924,993410,990307\n", + "Ground Truth: 995755,997437,993630,993484,995244,998649,994894,996630,992296,997320,999740,991999,998246,991991,994814,995435,994359,999599,997304,998855,999708,993696,996053,996436,997305,995841,990949,990786,993412,996075\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 91\n", + "Returned IDs: 998105,994307,993053,997790,991021,993906,996028,995823,991451,996147,990096,995586,995295,993289,992480,995456,995401,998677,999444,994689,996062,996001,991685,994280,992408,990532,992072,993433,998382,993561\n", + "Ground Truth: 998105,994307,993053,997790,991021,993906,996028,995823,991451,996147,990096,995586,995295,993289,992480,995456,995401,998677,999444,994689,996062,996001,991685,994280,993778,992408,990532,992072,993433,998382\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 93\n", + "Returned IDs: 998779,991743,996727,993412,996976,993673,991268,992087,995274,997640,999026,998649,999074,992511,997355,995034,992685,998862,997429,992147,998495,990406,994301,995616,997799,992637,992630,995755,996918,990949\n", + "Ground Truth: 998779,991743,996727,993412,996976,990401,993673,991268,992087,995274,997640,999026,998649,999074,992511,997355,995034,992685,998862,997429,992147,998495,990406,994301,995616,996438,997799,992637,992630,995755\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 100\n", + "Returned IDs: 996351,998837,994703,998363,991193,999738,993708,994789,995515,997496,993896,992384,995489,991840,990138,990490,991609,993143,990134,999453,991485,999613,990809,998657,998380,994001,992611,993577,994922,990310\n", + "Ground Truth: 996351,998837,994703,998363,991193,999738,993708,994789,995515,997496,993896,992384,995489,991840,990138,990490,991609,993143,990134,999453,991485,999613,990809,998657,998380,994001,992611,993577,994922,991930\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 102\n", + "Returned IDs: 999639,998793,992824,996196,997775,994270,992306,990538,992802,999908,996742,994560,990084,997547,992030,995631,994305,991923,995955,997215,990720,999451,997864,990755,993077,999564,992190,991432,992458,991910\n", + "Ground Truth: 999639,998793,992824,996196,997775,994270,992306,990538,992802,999908,996742,994560,990084,997547,992030,995631,994305,991923,995955,997215,990720,999451,997864,990755,993077,999564,992190,990409,991432,990146\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 104\n", + "Returned IDs: 997554,999453,995216,998495,996147,990845,996001,999402,992979,993412,994531,996368,991585,996810,991790,993003,992792,992334,990441,993561,991943,991967,996759,990662,992364,997031,993289,996911,995289,999866\n", + "Ground Truth: 997554,999453,995216,997937,998495,996147,990845,996001,999402,992979,993412,994531,998166,996368,991585,996810,991790,993003,992792,992334,990441,993561,991943,991967,996759,990662,992364,997031,993289,996911\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 105\n", + "Returned IDs: 991873,997913,991090,993834,999982,992822,992096,992228,998500,991626,995802,993796,997483,991616,992066,999165,992742,997320,990650,993616,997470,995513,996959,990182,997321,997759,990588,999710,998090,995949\n", + "Ground Truth: 991873,997913,991090,993834,999982,992822,999247,992096,992228,998500,993048,991626,995802,993796,997483,991616,992066,999165,992742,997320,990650,993616,997470,995513,996959,990182,997321,997759,990588,999710\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 107\n", + "Returned IDs: 994916,995274,994964,999144,999026,997015,999437,998779,992630,990535,993289,998951,993168,996959,997918,992047,994197,993492,991467,992216,995755,999596,992384,999374,997002,993484,990958,995336,990957,997320\n", + "Ground Truth: 994916,995274,994964,999144,999026,997015,999437,998779,992630,990535,993289,998951,993168,996959,997918,992047,997920,994197,993492,991467,999916,992216,995755,999596,992384,999374,997002,993484,990958,995336\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 108\n", + "Returned IDs: 994379,997319,991973,996411,990223,995611,990675,995819,991810,997822,991943,995289,994382,993909,993831,998156,995456,991350,992561,997732,993561,995633,994783,998750,990889,996906,995020,998765,997797,990277\n", + "Ground Truth: 994379,997319,991973,996411,990223,995611,990675,995819,999627,991810,997822,991943,995289,994382,993909,993831,998156,995456,991350,992561,997732,993561,995633,994783,998750,996607,990889,996906,995020,998765\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 109\n", + "Returned IDs: 999898,997028,996996,991311,994541,993695,997410,997920,993472,990974,997345,991339,998302,994783,996939,999343,998169,998039,998122,990210,993807,995384,999009,996062,996219,990505,997878,991776,992120,996216\n", + "Ground Truth: 999898,997028,996996,991311,994541,993695,997410,997920,993472,990974,997345,991339,991923,998302,994783,996939,999343,998169,998039,998122,990210,992931,993807,995384,999009,996062,996219,990505,995620,997878\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 110\n", + "Returned IDs: 997028,990557,996001,996219,999948,994777,998039,991498,999058,998765,995053,991930,994964,992446,993885,990752,994487,995102,997876,991523,991776,991212,997320,997332,996318,992182,991184,991921,995216,999617\n", + "Ground Truth: 997028,990557,996001,997381,996219,999948,994777,998039,991498,999058,998765,994061,995053,991930,995424,994964,992446,993885,990752,994487,995102,997876,991523,991776,991212,997320,997332,996318,992182,991184\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 112\n", + "Returned IDs: 995981,992470,992874,992384,995701,995792,990862,996001,998721,993432,993848,998246,991045,990588,998445,995467,994879,998549,993342,998921,995158,992167,997984,991459,997437,991245,996714,995336,995886,996075\n", + "Ground Truth: 993154,995981,992470,992874,992384,995701,995792,990862,996001,998721,993432,993848,999904,998246,991045,990588,998445,995467,994879,998549,993342,999977,998921,995158,997984,992167,991459,997437,991245,996714\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 113\n", + "Returned IDs: 996425,994819,994334,995453,997918,994804,994088,996046,997002,993465,996903,994362,999964,990995,995200,999277,995373,993407,995444,995374,994446,999089,999595,991603,990791,995928,998530,991942,997637,996864\n", + "Ground Truth: 996425,994819,994334,995453,997918,994804,994088,996046,997002,993465,996903,994362,999964,990995,995200,999277,995373,995444,993407,995374,994446,999089,999595,991603,990791,995928,998530,991942,997637,992260\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 116\n", + "Returned IDs: 998523,994922,992907,993355,996840,990043,993237,990334,997876,990332,990448,995004,999453,999620,997918,990432,990925,991510,996196,990538,990483,992612,996438,991609,997496,998401,999076,998837,993143,996402\n", + "Ground Truth: 998523,994922,992907,993355,996840,990043,993237,990334,997876,990332,990448,995004,999453,999620,997918,990432,990925,991510,996196,990538,990483,992612,996438,991609,997496,998401,995401,999076,998837,993143\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 117\n", + "Returned IDs: 996194,999601,998823,993014,994541,995695,990521,990169,995421,996396,994119,996429,998934,994280,995401,999496,994051,998546,994817,997316,990011,998250,992052,998968,991245,992313,992131,996633,998817,992437\n", + "Ground Truth: 996194,999601,998823,993014,994541,995695,990521,990169,995421,996396,994119,998348,996429,998934,994280,995401,999496,994051,998546,994817,997316,990011,998250,992052,998968,991245,992313,992131,996633,998817\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 119\n", + "Returned IDs: 995614,993426,990191,997873,997820,996516,997292,990855,994415,998921,997483,998721,996387,999109,999271,994618,995623,996296,996771,999792,994233,992853,991131,995306,993484,993785,995119,998445,996135,997550\n", + "Ground Truth: 995614,993426,990191,997873,997820,996516,997292,990855,994415,998921,997483,998721,996387,999109,999271,994618,995623,996296,996993,996771,999792,994233,992853,991131,995306,993484,993785,995119,998445,996135\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 127\n", + "Returned IDs: 998634,996001,997320,998246,993484,992839,991787,993053,994964,993280,991155,994961,997332,999099,990585,990146,993168,993885,991498,995678,999453,991721,996219,990772,990096,993389,990269,991732,998927,991609\n", + "Ground Truth: 998634,996001,997320,998246,993484,992839,991787,993053,994964,993280,991155,994961,997332,999099,990585,990146,993168,993885,991498,995678,999453,991721,991184,996219,990772,990096,993389,993154,990269,991732\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 128\n", + "Returned IDs: 997809,996546,991923,992718,998975,999380,993681,998331,991977,996163,990908,994398,993786,995138,993572,995986,994852,996658,992170,996158,995578,991157,999387,993975,998026,996894,995737,992349,996585,999806\n", + "Ground Truth: 997809,996546,991923,992718,998975,999380,993681,998331,991977,996163,990908,994398,993786,995138,993572,995986,994852,996658,992170,996158,995578,991157,998921,999387,993975,998026,996894,995737,992349,996585\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 129\n", + "Returned IDs: 992255,990650,996876,995794,990114,992964,992822,991873,998947,997451,999948,995485,997102,997948,998003,993083,997966,991616,999110,998434,990742,990066,995109,998540,990699,994461,990311,991178,995904,990030\n", + "Ground Truth: 992255,990650,996876,995794,990114,992964,992822,991873,998947,997451,999948,995485,997102,997948,998003,993083,997966,991616,993048,999110,995367,998434,990742,990066,995109,998540,990699,994461,991396,990311\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 131\n", + "Returned IDs: 998787,993366,994834,990952,995393,999373,998565,990167,994833,995326,999555,996553,998959,996408,990358,995299,990173,995231,992282,995835,993528,993560,994756,994374,993014,998536,995168,994682,994150,993948\n", + "Ground Truth: 998787,993366,994834,990952,995393,999373,998565,990167,994833,995326,999555,996553,998959,996408,990358,995299,990173,995231,992282,995835,993528,993560,994756,991313,994374,993014,998536,995168,994682,994150\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 132\n", + "Returned IDs: 999236,996716,992790,999820,998723,991011,993281,996371,996067,992179,993504,994946,998303,996062,998401,992576,995401,993071,998045,994113,991857,993979,992367,997331,996220,994188,997002,994964,993492,995289\n", + "Ground Truth: 999236,996716,992790,999820,998723,991011,993281,996371,996067,992179,993504,994946,998303,996062,998401,992576,995401,993071,998045,994113,991857,990294,993979,992367,997331,996220,994188,990244,997002,994964\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 133\n", + "Returned IDs: 999503,997790,993055,999840,994280,997926,993489,991397,995203,992862,997897,992521,990399,990005,992958,998250,990802,997918,998303,991661,994042,994531,991630,994946,991332,998933,992747,996657,998697,996368\n", + "Ground Truth: 999503,997790,993055,999840,994280,997926,993489,991397,995203,992862,992979,997897,992521,990399,990005,992958,998250,990802,997918,998303,991661,994042,994531,991630,994946,991332,998933,992747,996657,993143\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 134\n", + "Returned IDs: 991227,992558,992052,993787,993478,992933,998536,996918,996531,995783,994280,995214,994972,995408,990956,992318,993673,991052,998104,998397,993121,992997,993747,996438,996658,995401,996930,997897,993727,996328\n", + "Ground Truth: 991227,992558,992052,993787,993478,992933,998536,996918,996531,995783,994280,995214,994972,995408,990956,992318,993673,991052,998104,998397,993121,992997,993747,996438,996658,995401,996930,997897,993727,992521\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 135\n", + "Returned IDs: 994406,992767,997799,998431,995403,997391,999960,990337,998779,997634,993026,994474,998080,994671,996727,996903,995374,995453,997012,995034,997157,997754,997640,993412,990647,993289,996172,995893,995607,998915\n", + "Ground Truth: 994406,992767,997799,998431,995403,997391,999960,990337,998779,997634,993026,994474,998080,994671,996727,996903,995374,996047,995453,997012,995034,997157,997754,997640,993412,990647,993289,996172,995893,995607\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 136\n", + "Returned IDs: 991324,999740,992698,993725,990011,990484,991390,994830,995756,990711,992479,997142,990743,994751,998710,990089,995002,991885,996358,999922,996633,997714,992664,991911,995435,992679,990439,992480,996001,997933\n", + "Ground Truth: 991324,999740,992698,993725,990011,990484,991390,994830,995756,990711,992479,997142,990743,994751,998710,990089,995002,991885,999555,996358,999922,996633,997714,992664,991911,995435,992679,990439,992480,996001\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 138\n", + "Returned IDs: 999350,994864,994488,995167,996001,992169,996406,999179,997679,991212,992182,998122,994510,991596,990557,993931,999130,999304,999623,994467,994119,990434,997332,996075,997395,993032,994416,998725,996775,991066\n", + "Ground Truth: 999350,994864,994488,995167,996001,992169,996406,999179,997679,991212,999679,992182,996803,998122,994510,991596,993931,990557,999130,999304,999623,994467,999099,994119,990434,997332,996075,997395,993032,994416\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 139\n", + "Returned IDs: 994168,992469,999500,995289,990126,997723,993656,990930,999586,996811,998913,995562,994783,991154,990340,993747,994742,996219,995095,994759,998495,997797,990523,998156,990266,993642,996116,999977,990399,998855\n", + "Ground Truth: 994168,992469,999500,995289,990126,997723,993656,990930,999586,996811,998913,995562,994783,991154,990340,993747,993400,994742,996219,995095,994759,997864,998633,998495,997797,990523,998156,990266,993642,996116\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 144\n", + "Returned IDs: 995515,992835,990155,998244,996419,993582,993028,994216,998837,990142,991149,999053,990313,993237,991999,997591,994612,994281,992020,990652,997403,994475,994497,990827,997465,990043,992662,994898,996485,991196\n", + "Ground Truth: 995515,992835,990155,998244,996419,993582,993028,994216,998837,990142,991149,999053,990313,993237,991999,997591,994612,994281,992020,990652,997403,990943,994475,994497,990827,997465,990043,992662,994898,996485\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 150\n", + "Returned IDs: 992678,996848,998067,997915,991044,990331,991747,992097,999980,992521,997675,992569,998045,995268,991103,990314,996368,996277,996505,998060,993256,992367,996719,995963,992187,994861,999402,997723,993313,994817\n", + "Ground Truth: 992678,996848,998067,997915,991044,990331,991747,992097,999980,992521,997675,992569,998045,995268,998021,991103,990314,996368,996277,996505,998060,993256,992367,996719,995963,992187,994861,999402,997723,993313\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 151\n", + "Returned IDs: 999578,992448,994409,993468,992660,992801,996618,997832,999043,996038,999530,998714,991699,997868,996813,997406,995139,998045,998979,999508,996607,991027,995507,994554,993433,996769,991671,993561,998545,999940\n", + "Ground Truth: 999578,992448,994409,993468,992660,992801,996618,997832,999043,996038,999530,998714,991699,997868,996813,997406,995139,998045,998979,999508,996607,991027,995507,994554,993433,996769,991671,993561,993280,998545\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 153\n", + "Returned IDs: 992239,995637,990264,990814,991981,991590,999924,999767,992713,994891,993080,997494,994968,991428,990516,991769,995012,994094,998999,992318,997698,990364,990270,992437,996728,998869,999942,992827,996658,994742\n", + "Ground Truth: 992239,995637,990264,990814,991981,991590,999924,999767,992713,994891,993080,997494,994968,991428,990516,991769,995012,994094,998999,992318,997698,990364,990270,992437,996728,998869,999942,990853,992827,996658\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 154\n", + "Returned IDs: 994168,990080,990266,994131,998633,991943,996093,999586,991255,994742,997723,999587,998156,991981,993080,993717,992582,999453,992169,991585,995295,994129,990957,998980,996867,991185,994431,996809,998256,995702\n", + "Ground Truth: 994168,990080,990266,994131,992655,998633,991943,996093,999586,991255,994742,997723,999587,998156,991981,993080,993717,992582,999453,992169,997360,991585,991334,995295,994129,990957,998980,996867,991185,994431\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 156\n", + "Returned IDs: 999267,993637,991128,993053,995283,995659,990089,990368,999338,995274,990629,996701,995345,991412,991992,993410,999806,992636,997691,997052,994386,995638,998302,995467,998021,999343,997340,999446,996368,998215\n", + "Ground Truth: 999267,993637,991128,993053,995283,995659,990089,990368,999338,995274,990629,996701,990133,995345,991412,991992,993410,999806,992636,997691,997052,994386,995638,998302,995467,998021,999343,990260,997340,999446\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 158\n", + "Returned IDs: 990890,996219,991928,998899,992931,991249,998540,994831,994069,999300,990545,999068,999343,995942,994964,992330,991593,991743,996343,997913,993282,990404,994486,998550,999620,997305,990210,998837,990444,996646\n", + "Ground Truth: 990890,996219,991928,998899,992931,991249,998540,994831,994069,999300,990545,999068,999343,995942,994964,992330,991593,991743,996343,997913,993282,990404,994486,998550,990666,999620,991006,995210,997305,990210\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 159\n", + "Returned IDs: 992094,994119,996245,997944,998150,994177,994193,990226,990527,994280,999461,992703,992052,998755,993037,992437,994968,997309,997897,993483,993915,992652,997056,993288,995401,998355,995408,996277,990784,996775\n", + "Ground Truth: 992094,994119,996245,997944,998150,994177,994193,990226,990527,994280,999461,992703,992052,998755,993037,992437,990790,994968,997309,997897,993483,993915,992652,997056,993288,995401,998355,995408,996277,990784\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 160\n", + "Returned IDs: 992604,993754,995807,995527,999662,999445,997781,990915,990540,995953,991051,990657,990729,991204,991110,991571,994187,997918,997212,995373,994753,998858,991987,996043,999306,995805,997391,997675,995105,998697\n", + "Ground Truth: 992604,993754,995807,995527,999662,999445,997781,990915,990540,995953,991051,990657,990729,998297,991204,991110,991571,994187,997918,997212,995373,994753,998858,991987,996043,997158,999306,995805,997391,997675\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 161\n", + "Returned IDs: 998294,996504,999765,990917,997913,994266,998921,993518,991406,997984,991895,990760,999263,993669,998500,995755,990925,994978,990829,995090,998987,990936,997192,990595,996560,997371,998540,995793,998344,992789\n", + "Ground Truth: 998294,996504,999765,990917,997913,994266,998921,993518,991406,997984,991895,990760,999263,993669,998500,995755,990925,994978,990829,995090,998987,990936,997192,990595,996560,997371,998540,995793,998344,999647\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 162\n", + "Returned IDs: 995110,993478,991103,996505,995032,998237,998397,995783,999646,991747,995401,997675,995701,993727,990537,995521,991413,991010,990956,992457,997918,997752,991052,995184,991054,994232,998045,990872,992790,993412\n", + "Ground Truth: 995110,993478,991103,996505,995032,998237,998397,995783,999646,991747,995401,997675,995701,993727,990537,995521,991413,991010,990956,992457,993738,991630,997918,997752,991052,995184,991054,994232,992260,998045\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 163\n", + "Returned IDs: 998914,990678,996556,996028,998355,997380,995528,999891,998630,990226,991426,996190,993725,992521,994765,995317,996505,996016,992488,996728,994119,998325,999479,992480,993820,995449,995456,998982,999516,991747\n", + "Ground Truth: 998914,990678,996556,996028,998355,997380,995528,999891,998630,990226,991426,996190,993725,992521,994765,995317,996505,996016,992488,996728,992322,994119,998325,999479,992480,993820,995449,995456,993336,998982\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 170\n", + "Returned IDs: 999430,995332,995107,991179,991961,991458,994897,998416,998950,990377,993499,995560,996216,993807,995895,997331,990091,995750,993617,993079,990769,992780,997004,993479,996944,999210,992567,998157,996370,991382\n", + "Ground Truth: 999430,995332,995107,991179,991961,991458,994897,998416,998950,990377,993499,995560,996216,993807,995895,997331,990091,995750,993617,993079,990769,992780,997004,993479,996944,999210,992567,993335,998157,996370\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 171\n", + "Returned IDs: 996219,994990,994964,993378,991075,996052,994843,992047,996593,997028,999453,995096,993342,996001,998039,990448,992780,990366,997920,994777,994472,999352,992261,994922,990091,993648,999437,999344,998464,998495\n", + "Ground Truth: 996219,994990,994964,993378,991075,996052,994843,997616,992047,996593,997028,999453,995096,993342,996001,998039,990448,992780,990366,997920,994777,994472,999352,992261,994922,990091,993648,998633,999437,999344\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 172\n", + "Returned IDs: 990016,998105,992201,993588,994145,993053,995097,990435,990497,996147,990536,994307,998548,991075,990662,994830,995289,994925,990771,996974,999460,993831,998097,990811,996001,993412,995823,996411,998950,998495\n", + "Ground Truth: 990016,998105,992201,993588,994145,993053,995097,990435,990497,996147,990536,994307,998548,991075,990662,994830,995289,994925,990771,996974,999460,998165,993831,998097,990811,996001,993412,995823,996411,998950\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 175\n", + "Returned IDs: 994894,990689,997192,998974,995274,999444,994266,990957,993186,990498,997035,998326,997437,998697,991630,990337,995536,996699,992811,990231,992357,997320,990958,996785,998649,993412,990799,998170,990146,992692\n", + "Ground Truth: 994894,990689,997192,998974,995274,999444,994266,990957,993186,990498,997035,998326,997437,998697,991630,990337,990862,995536,996699,992811,990231,992357,997320,990958,996785,998649,993412,990799,998170,990146\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 178\n", + "Returned IDs: 994254,997809,998686,990487,991977,992889,997407,991923,998046,990776,995691,994165,998243,994760,990343,996068,999766,998975,993839,990522,993683,997914,998357,993572,994010,995965,995506,996895,992645,998118\n", + "Ground Truth: 994254,997809,998686,990487,991977,992889,997407,991923,998046,990776,995691,994165,998243,994760,990343,996068,999766,998975,993839,990522,993683,997914,998357,993572,994010,995965,995506,999913,996895,992645\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 179\n", + "Returned IDs: 990155,993280,999453,996810,992395,998734,990772,996001,991375,991951,995988,990142,994536,996915,996732,997876,991517,998567,995216,990778,995130,999076,991840,991255,991248,999019,997320,998605,991999,992944\n", + "Ground Truth: 990155,993280,999453,996810,992395,998734,990772,996001,991375,991951,995988,990142,994536,996915,996732,997876,991517,998567,995216,990778,995130,999076,991840,998523,991255,991248,999019,997320,999723,993154\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 181\n", + "Returned IDs: 995895,996706,997442,998593,993805,993391,995438,992726,990844,998993,990133,999430,994431,994351,994742,996809,999435,999179,996930,998934,998989,994830,993335,997723,992864,997331,998055,994382,996830,992850\n", + "Ground Truth: 995895,996706,997442,998593,993805,993391,995438,992726,990844,998993,990133,999430,994431,994351,994742,996809,999435,999179,996930,998934,999587,998989,998990,994830,993335,997723,992864,997331,998055,994382\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 183\n", + "Returned IDs: 992002,991507,998162,995493,996607,998910,999054,997466,992448,990163,995458,996987,993046,990295,991671,991585,992260,994013,994865,998948,990924,990804,996042,996815,991603,998283,999475,995216,995766,993561\n", + "Ground Truth: 992002,991507,998162,995493,996607,998910,999054,997466,992448,990163,995458,996987,993046,990295,991671,991585,997529,992260,994013,994865,998948,990924,993883,990804,996042,996815,991603,998283,999475,995216\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 184\n", + "Returned IDs: 996689,990328,990600,994255,992445,999025,995456,994689,992052,995295,991277,992068,996028,997329,991451,995832,997309,994280,998959,993560,997420,995166,990532,998589,995823,995273,997897,998182,995203,994946\n", + "Ground Truth: 996689,990328,990600,994255,992445,999025,995456,994689,992052,995295,991277,992068,996028,997329,996110,991451,995832,997309,994280,998959,993560,997420,995166,990532,998589,995823,995273,997897,998182,995203\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 187\n", + "Returned IDs: 997391,997918,994334,990215,996282,994819,993492,995886,996903,999562,997702,992979,992179,999820,995401,992211,993512,997782,994088,994319,992428,999144,994916,992933,994906,998892,990561,998011,994670,996658\n", + "Ground Truth: 997391,997918,994334,990215,996282,994819,993492,995886,996903,999562,997702,992979,992047,992179,999820,995401,992211,993512,997782,994088,994319,992428,999144,994916,992933,994906,998892,990561,998011,994670\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 188\n", + "Returned IDs: 997106,992190,998409,995256,990675,992804,993791,994852,991958,991923,999639,997864,994043,993342,991208,995701,996368,993283,992167,991530,997407,990622,998079,995138,997739,995898,991350,994105,996894,997577\n", + "Ground Truth: 997106,992190,998409,995256,990675,992804,993791,998974,994852,991958,991923,999639,997864,994043,993342,991208,995701,996368,993283,992167,994796,991530,997407,990622,998079,995138,997739,995898,991350,994105\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 191\n", + "Returned IDs: 999172,991585,998060,994013,997554,999570,999402,995544,996147,993377,993053,996001,993217,993209,991185,990887,997059,990331,997529,992260,997566,994203,996752,993561,992450,995873,998162,993021,995216,994488\n", + "Ground Truth: 999172,991585,998060,994013,997554,999570,999402,995544,996147,993377,993053,996001,993217,993209,991185,990887,997059,990331,997529,992260,997566,994203,996752,993561,992450,993166,995873,998162,993021,995216\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 192\n", + "Returned IDs: 991372,993696,990628,994722,990885,996809,996296,991109,992047,992635,997098,996452,990697,991172,998734,994922,995210,999219,992927,995530,996875,995596,996913,999337,996350,998790,993504,993038,991425,990627\n", + "Ground Truth: 991372,993696,990628,994722,990885,996809,996296,997166,991109,992047,992635,997098,996452,990697,991172,998734,994922,995210,999219,992927,995530,994859,996875,995596,996913,998303,999337,996350,998790,993504\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 199\n", + "Returned IDs: 990382,997476,991581,999827,996062,992314,993981,994510,998095,994463,998436,992161,993289,990899,990520,991930,997790,997642,993931,998158,992169,995954,997453,991596,995636,990557,990212,995185,991644,994411\n", + "Ground Truth: 990382,997476,991581,999827,996062,992314,993981,994510,998095,994463,998436,992161,993289,990899,990520,991930,997790,996913,997642,993931,998158,992169,995954,997453,991596,995636,990557,990212,995185,991644\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 200\n", + "Returned IDs: 997320,997370,996811,992944,997305,998950,991498,995886,993018,994266,995372,992041,990949,995250,990772,999453,995203,997301,995927,993154,991801,999210,993260,990532,997470,995438,997913,993241,995841,997371\n", + "Ground Truth: 997320,997370,996811,992944,997305,998950,991498,995886,993018,994266,995372,992041,990949,995250,990772,999453,995203,997301,995552,990917,995927,993154,991801,999210,993260,990532,997470,995438,997913,993241\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 203\n", + "Returned IDs: 992927,993260,994988,990429,994964,994537,996193,990786,996053,997320,999216,996576,993055,990885,999176,992069,998568,997918,993783,991795,991675,992571,995536,992497,994789,992225,999711,994552,998299,999800\n", + "Ground Truth: 992927,993260,994988,990429,994964,994537,996193,990786,996053,997320,999216,996576,999362,993055,990885,999176,992069,998568,994984,997918,993783,991795,991675,992571,995536,992497,994789,992225,999711,994552\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 204\n", + "Returned IDs: 990487,994117,996391,998370,995896,991350,992929,997554,999587,990211,997868,994793,994207,996368,996379,993998,994809,996654,996657,997723,990314,998990,996127,991973,995873,998686,996181,993831,991585,999639\n", + "Ground Truth: 990487,994117,996391,998370,995896,991350,992929,997554,999587,990211,997868,994793,994207,996368,996379,993998,994809,996654,996657,997723,990314,998990,996127,991973,995873,998686,996181,993831,991585,992423\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 208\n", + "Returned IDs: 994273,999415,993127,995122,992449,995423,995702,994013,999745,995701,999054,991636,996867,995182,996897,995685,998636,990082,995905,997077,994954,990152,994409,998776,995401,997975,998518,997674,996232,993856\n", + "Ground Truth: 994273,999415,993127,995122,992449,995423,995702,994013,999745,995701,999054,991636,996867,995182,996897,995685,998636,990082,995905,997077,994954,990152,994409,998776,995401,997975,998518,997674,995303,996232\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 211\n", + "Returned IDs: 999453,994167,991987,990999,991811,996161,991894,995596,996001,991459,999219,995899,990949,996657,990957,991445,995855,992979,996368,995281,990542,997320,997258,992562,991835,993848,995866,999639,996744,992384\n", + "Ground Truth: 999453,994167,991987,990999,998567,991811,996161,991894,995596,996001,991459,993154,999219,995899,990949,996657,990957,991445,995855,992979,996368,995281,990542,997320,997258,992562,991835,993848,995866,999639\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 212\n", + "Returned IDs: 992824,995459,990894,996380,999639,991272,995631,995503,991064,996742,990273,991432,996663,999908,998061,999387,997547,996894,991817,995728,993055,999674,993839,994560,993225,999346,998720,990061,992306,995724\n", + "Ground Truth: 992824,995459,990894,996380,999639,991272,995631,995503,991064,996742,990273,991432,996663,999908,998061,999387,997547,996894,991817,995728,993055,999674,993839,994560,993225,999346,992752,998720,990061,992306\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 213\n", + "Returned IDs: 993879,995080,997675,995521,992801,991904,996198,990244,999082,994170,994409,996505,995401,993030,990527,996277,998045,993561,992448,995873,997822,998049,997832,991603,996472,994677,997020,997766,992260,999461\n", + "Ground Truth: 993879,995080,997675,995521,992801,991904,996198,990244,999082,994170,994409,996505,995401,993030,990527,996277,998045,993561,992448,995873,997822,998049,997832,991603,996472,994677,997020,997766,992260,990140\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 215\n", + "Returned IDs: 996062,998122,998495,999898,994869,990091,990194,990096,997242,990434,999827,991893,993106,999555,996219,996001,996028,995655,995536,992056,994266,995562,999139,990167,990382,990796,998593,994200,997790,996677\n", + "Ground Truth: 996062,998122,998495,999898,994869,990091,990194,990096,997242,990434,999827,991893,993106,999555,996219,996001,996028,998861,995655,995536,992056,994266,995562,999139,990167,990382,990796,998593,994200,997790\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 216\n", + "Returned IDs: 997429,995184,993238,991856,990958,992357,993648,994835,999533,995963,994266,995034,996861,992008,993168,995800,994956,998458,990925,995927,997852,995783,993885,992526,999374,992147,995884,990399,997309,991468\n", + "Ground Truth: 997429,995184,993238,991856,990958,992357,993648,994835,999533,995963,994266,995034,996861,991714,992008,993168,995800,994956,998458,990925,995927,997852,995783,993885,992526,999374,992147,995884,990399,997309\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 218\n", + "Returned IDs: 990711,990392,997529,991703,993053,998862,993725,996411,994730,995002,992156,993240,998913,999302,999377,995633,994742,990701,998495,998779,994830,995401,990435,993166,993264,995096,992680,996408,991390,995601\n", + "Ground Truth: 990711,990392,997529,991703,993053,998862,993725,996411,994730,995002,992156,993240,998913,999302,999377,995633,994742,990701,998495,998779,994830,995401,990435,993166,993264,995096,992680,996408,993856,991390\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 220\n", + "Returned IDs: 994643,998536,997752,993232,999117,998383,999155,990551,995625,993995,996809,992288,998695,996797,991532,992558,991630,997666,995963,990941,994063,991571,996043,997131,991675,994742,995401,992604,992809,996395\n", + "Ground Truth: 994643,998536,997752,993232,999117,998383,999155,990551,995625,993995,996809,992288,998695,996797,991532,992558,991630,997666,995963,990941,994063,991571,996043,993077,997131,991675,994742,995401,992604,992809\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 221\n", + "Returned IDs: 995596,995515,993143,991840,991848,999122,999784,999806,996840,990432,994709,997403,993748,995443,996219,995365,999676,992979,999144,991395,991485,997135,997379,999613,991977,992309,996062,997918,994916,993237\n", + "Ground Truth: 995596,995515,993143,991840,991848,999122,999784,999806,996840,990432,994709,997403,993748,995443,996219,995365,999676,992979,999144,991395,991485,992047,997135,997379,999613,991977,992309,996062,997918,994916\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 223\n", + "Returned IDs: 997030,990096,994717,994643,994413,993666,992638,993115,998257,991309,994454,999898,990815,997077,997154,991190,998277,999555,997211,993566,992369,995218,994677,998045,991596,991188,997568,990361,994569,998536\n", + "Ground Truth: 997030,990096,994717,994643,994413,993666,992638,993115,999530,998257,991309,994454,999898,990815,997077,997154,991190,997871,998277,999555,997211,993566,992369,995218,994677,998045,991596,991188,997568,990361\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 224\n", + "Returned IDs: 990049,995609,996194,999528,995043,995551,994326,993678,991241,996232,990417,993181,990011,996435,992566,994541,992068,999830,998769,992473,990121,993245,994317,995475,995537,993315,996867,992563,995317,992951\n", + "Ground Truth: 990049,995609,996194,999528,995043,995551,994326,993678,991241,996232,990417,993181,990011,996435,992566,994541,992068,999830,998769,992473,990121,993245,994317,995475,995537,993315,997906,990153,996867,992563\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 227\n", + "Returned IDs: 992511,998974,996252,997437,993457,990337,999867,994894,995838,997174,996917,997728,999599,993978,995755,998649,994916,990957,997320,996648,996192,995616,995274,995963,997304,994630,995374,991995,992531,993049\n", + "Ground Truth: 992511,998974,996252,997437,993457,990337,999867,994894,995838,997174,996917,997728,993222,999599,993978,995755,998649,994916,990957,997429,997320,996648,996192,995616,995274,995963,997304,994630,995374,991995\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 228\n", + "Returned IDs: 998060,993487,994203,992352,994106,992098,990573,992000,991057,996075,994488,996001,995079,995544,991212,994013,990662,990314,999172,995515,997998,998593,996898,995873,994954,992569,995640,992097,991394,999590\n", + "Ground Truth: 998060,993487,994203,992352,994106,992098,990573,992000,991057,996075,994488,996001,995079,995544,991212,994013,990662,990314,999172,996043,995515,997998,998593,996898,995873,994954,992569,995640,992097,991394\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 229\n", + "Returned IDs: 992558,990279,997547,997124,996196,990538,999715,995336,999908,995886,996606,993948,994334,997859,992718,990989,992971,991157,991052,996609,996918,999860,992857,990930,995552,996721,994163,993363,990343,992521\n", + "Ground Truth: 992558,990279,997547,997124,996196,990538,999715,995336,999908,995886,996606,993948,994334,997859,992718,990989,992971,991157,991052,996609,996918,999860,992857,990930,995552,996721,994163,993363,990343,991498\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 230\n", + "Returned IDs: 994266,995184,997320,997913,990231,998170,992546,991256,999647,992330,995701,993853,997305,995417,996296,992637,996075,997554,999761,990958,996661,993834,997449,990650,996083,990535,993168,992748,991045,994476\n", + "Ground Truth: 994266,995184,997320,997913,990231,998170,992546,991256,999647,992330,995701,993853,999041,997305,995417,996296,992637,996075,997554,999761,990958,996661,993834,997449,990650,996083,993168,990535,992748,991045\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 231\n", + "Returned IDs: 998913,994783,998823,998370,998196,995551,991428,994742,992566,991981,994806,994692,995289,995397,995348,992415,995948,993939,995030,992827,997630,996411,991330,995633,998543,995300,994110,998028,996609,994587\n", + "Ground Truth: 998913,994783,998823,998370,998196,995551,991428,994742,992566,991981,994806,994692,995289,995397,990846,995348,992415,995948,993939,995030,992827,997630,996411,991330,995633,998543,995300,994110,998028,996609\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 233\n", + "Returned IDs: 992981,992445,997806,996328,994336,990956,996728,996861,998017,994125,993055,990005,997420,997884,996211,996918,997506,994142,999775,993948,996189,998959,993560,992293,990961,993877,998495,998067,995516,994280\n", + "Ground Truth: 992981,992445,997806,996328,994336,990956,996728,996861,998017,992979,994125,993055,990005,997420,997884,996211,996918,997506,994142,999775,993948,996189,998959,993560,992293,990961,993877,998495,998067,995516\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 236\n", + "Returned IDs: 998971,991596,992673,999663,999139,994510,996406,997127,999898,996985,996706,996809,999555,996733,991420,997980,998149,999435,992365,998006,996209,994335,992234,997242,999498,990226,995104,997506,991589,994103\n", + "Ground Truth: 998971,991596,992673,999663,999139,994510,996406,997127,999898,996985,996706,996809,999555,996733,991420,997980,998149,999435,992365,998006,996209,994335,992234,997242,999498,990226,995104,997506,991589,999783\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 240\n", + "Returned IDs: 997397,999110,999523,997301,999471,992451,995623,999904,991916,993484,990146,995306,990546,998760,997470,990405,997820,993426,993083,997292,995264,997320,992066,999242,995045,991582,990182,990191,996516,993853\n", + "Ground Truth: 997397,999110,999523,997301,999471,992451,995623,999904,991916,993484,990146,995306,990546,998760,997470,990405,997820,993426,993083,997292,995264,997320,992066,999242,995045,999247,991582,990182,990191,996516\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 242\n", + "Returned IDs: 995659,999271,990585,997668,997286,990089,999338,990000,994942,994618,997292,997984,995841,997470,997301,996516,995306,992069,991503,997320,998721,998921,995372,991570,998923,998445,998461,993419,994964,997113\n", + "Ground Truth: 995659,999271,990585,997668,997286,990089,999338,990000,994942,994618,997292,997984,995841,997470,997301,996516,995306,992069,991503,997320,998721,998921,995372,991570,998923,998445,998461,993419,994964,992451\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 247\n", + "Returned IDs: 995473,992815,991045,992694,999616,998524,990588,997620,995981,991900,997304,992874,997437,993300,997320,993392,997913,998365,997353,999291,999725,998246,992074,990535,998353,995701,991745,992384,992927,994266\n", + "Ground Truth: 995473,992815,991045,992694,999616,998524,990588,997620,995981,991900,997304,992874,997437,993300,997320,993392,997913,998365,997353,999291,999725,998246,992074,998380,990535,998353,995701,991745,992384,992927\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 249\n", + "Returned IDs: 997529,995020,990711,998719,999500,992156,991189,994120,998495,994382,993412,993725,996411,997207,995633,990126,991881,991669,990435,991075,993439,998671,994730,993747,998779,998982,998862,990256,995131,990340\n", + "Ground Truth: 997529,995020,990711,998719,999500,992156,991189,994120,998495,994382,993412,993725,996411,997207,995633,990126,991881,991669,990435,991075,993439,997485,998671,994730,993747,998779,998982,993656,998862,990256\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 251\n", + "Returned IDs: 999826,997395,990790,999020,998080,994612,994170,996245,997184,993931,995807,997986,992879,998820,994770,995163,998049,992790,990243,997189,994280,998014,993149,992683,999967,991698,999898,995521,997918,993933\n", + "Ground Truth: 999826,997395,990790,999020,998080,994612,994170,996245,997184,993931,995807,997986,992879,998820,994770,997301,995163,998049,992790,990243,997189,994280,998014,993149,992683,999967,991698,999898,995521,997918\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 258\n", + "Returned IDs: 990827,990043,998983,998937,990849,997370,996840,998551,997789,995988,999620,998734,991196,994922,999898,990858,997445,995515,993282,998236,991906,991348,997255,990500,993647,992169,993143,993237,995443,990701\n", + "Ground Truth: 990827,990043,998983,998937,990849,997370,996840,998551,997789,995988,999620,994735,998734,991196,994922,999898,990858,997445,995515,993282,998236,991906,991348,997255,990500,993647,992169,993143,993237,995443\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 259\n", + "Returned IDs: 996219,993168,995876,993637,995001,999647,998343,993499,990173,999009,995689,998495,997331,999673,996067,990784,999430,991037,995562,996549,991758,996062,994279,991109,992589,995438,990323,991372,999748,992960\n", + "Ground Truth: 996219,993168,995876,995001,993637,999647,998343,993499,990173,999009,995689,998495,997331,999673,996067,990784,999430,997992,991037,995562,996549,991758,996283,996062,994279,991109,992589,995438,990323,991372\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 261\n", + "Returned IDs: 998917,997220,994617,994491,997136,995259,998782,998139,991727,995588,998983,994534,992255,994741,990822,991999,999469,992085,998928,995334,991806,991405,993340,990809,994528,998236,990890,994785,999147,995316\n", + "Ground Truth: 998917,997220,994617,994491,997826,997136,995259,998782,998139,991727,995588,998983,994534,992255,994741,990822,991999,999469,992085,998928,995334,999535,991806,991405,993340,990809,994528,998236,990890,994785\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 264\n", + "Returned IDs: 991137,996653,999533,998302,993845,997855,997320,999109,993579,996726,993066,997381,995569,995841,994978,999338,994332,999827,991714,997984,992318,993807,992228,995659,998445,992647,998797,998349,996729,990146\n", + "Ground Truth: 991137,996653,999533,998302,993845,997855,997320,997921,999109,993579,996726,990179,993066,997381,995569,995841,994978,999338,994332,999827,991714,997984,991787,992318,993807,992228,995659,998445,992647,998797\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 266\n", + "Returned IDs: 999356,998353,995981,998163,993853,998246,997728,992296,990453,994879,995755,990231,998662,992074,996075,993529,993186,990708,996609,994100,999761,996384,992848,990799,991895,999740,997192,998950,998332,991721\n", + "Ground Truth: 999356,998353,995981,998163,993853,998246,997728,992296,990453,994879,995755,990231,998662,991782,992074,996075,993529,993186,991913,990708,996609,994100,999761,996384,992848,990799,991895,999740,997906,997192\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 268\n", + "Returned IDs: 992944,995360,991848,993781,996829,990043,999784,998991,994543,994341,991013,999210,997135,993708,998950,993260,996193,991416,996802,998180,990182,995375,999122,996007,993365,997037,993633,997457,998726,993279\n", + "Ground Truth: 992944,995360,991848,993781,996829,990043,999784,998991,994543,994341,991013,999210,999577,997135,993708,998950,993260,996193,991416,996802,998180,990182,992168,995375,999122,996007,993365,997037,993633,997457\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 269\n", + "Returned IDs: 996429,997918,995701,998932,994276,995785,996217,992047,998303,991530,994832,994221,993260,990409,993055,995932,990930,998123,999384,994754,992168,998697,997897,990751,996196,993504,995203,992571,997586,990494\n", + "Ground Truth: 996429,997918,995701,998932,994276,995785,996217,992047,998303,991530,994832,994221,993260,990409,993055,995932,990930,998123,999384,994754,992168,998697,997897,990751,996196,993504,995203,994514,990425,992571\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 270\n", + "Returned IDs: 990530,992562,997925,999639,995701,995401,995005,999041,990588,993821,995886,990217,991154,993809,990930,996067,998200,991459,993834,997337,999680,996918,992933,999219,991609,992927,994510,997918,995932,994081\n", + "Ground Truth: 990530,992562,997925,999639,995701,995401,995005,999041,990588,993821,995886,990217,991154,993809,990930,996067,998200,991459,993834,997337,999680,996918,992933,999219,991609,992927,994510,997918,995636,995932\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 272\n", + "Returned IDs: 995208,996968,996867,998297,995401,990337,994794,999530,992169,993412,990432,995274,994409,998536,995373,997191,990561,994677,997871,998045,994301,995515,995374,993143,996727,998080,991052,997723,997319,996769\n", + "Ground Truth: 995208,996968,996867,998297,995401,990337,994794,999530,992169,993412,990432,995274,994409,998536,995373,990561,997191,994677,997871,998045,994301,995515,997775,997403,995374,993143,996727,998080,991052,996438\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 273\n", + "Returned IDs: 990072,999384,998861,997666,990096,997906,996623,991188,997685,991459,995780,998419,991471,993275,997265,999362,998623,993821,996209,998536,995701,990194,998383,999653,993561,996733,994105,994906,999827,999319\n", + "Ground Truth: 990072,999384,998861,997666,990096,997906,996623,991188,997685,991459,995780,998419,991471,993275,997265,999362,998623,993821,996209,998536,995701,990194,992977,998383,999653,993561,999272,996733,994105,994906\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 275\n", + "Returned IDs: 990020,997320,991989,990532,997661,997918,990207,993785,999219,991795,992950,991829,998634,998302,996646,999446,995852,996075,991311,993906,999444,994171,997227,997906,996735,994307,991984,999990,994081,993561\n", + "Ground Truth: 990020,997320,991989,990532,997661,997918,990207,993785,999219,991795,992950,991829,998634,998302,996646,999446,995852,996075,991311,993906,993412,999444,994171,997246,997227,997906,996735,994307,991984,999990\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 277\n", + "Returned IDs: 992546,990515,991431,993392,999180,992339,992994,997614,990958,994653,998246,990118,997305,994757,997437,996075,998797,991479,995981,994079,997320,990660,998562,997976,994776,997543,997470,991045,997913,993662\n", + "Ground Truth: 992546,990515,991431,993392,992583,999680,992238,999180,992339,992994,997614,990958,994653,998246,990118,997305,994757,997437,996075,998797,991479,995981,994079,997320,990660,998562,997976,994776,997543,997470\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 278\n", + "Returned IDs: 994427,990231,994403,996921,992969,993220,995697,992363,991743,998550,994171,996646,999710,994450,992119,993662,993278,997320,994699,994048,995351,990135,995942,999167,994192,997354,990910,996540,990066,990182\n", + "Ground Truth: 994427,990231,994403,996921,992969,993220,995697,992363,991743,998550,994171,996646,999710,994450,992119,993662,993278,997320,994699,991320,994048,995351,990135,995942,999167,994192,997354,990910,996540,990066\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 281\n", + "Returned IDs: 991269,993209,997023,993586,999957,995778,997723,991507,997544,994496,999054,998188,991639,991943,999415,999047,991140,998779,991136,999563,996867,992334,997975,990372,998295,994881,992582,995289,997832,990361\n", + "Ground Truth: 991269,993209,997023,993586,999957,996607,995778,997723,991507,997544,994496,999054,998188,991639,991943,999415,999047,991140,998779,997101,991136,999563,996867,992334,997975,990372,998295,992413,994881,992582\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 282\n", + "Returned IDs: 991820,993080,998720,991981,993342,995906,993809,998514,990860,996911,999444,995160,998593,995852,990532,999916,990866,998710,995002,999922,996001,991795,999647,997790,995961,995636,993906,994636,990622,992314\n", + "Ground Truth: 991820,993080,998720,991981,993342,995906,993809,998514,990860,996911,999444,995160,998593,995852,990532,996490,999916,990866,993290,999272,997969,998710,995002,999922,996001,991795,999647,997790,995961,995636\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 283\n", + "Returned IDs: 999912,998039,997920,996996,995384,995906,990737,997028,991565,997410,997529,990974,997345,995526,990315,999949,998169,993637,996219,994964,995222,994293,999261,990891,994783,999647,993378,993168,998382,999151\n", + "Ground Truth: 999912,998039,997920,996996,995384,995906,990737,997028,991565,997410,997529,990974,997345,995526,990315,999949,998169,993637,996219,994964,995222,994293,991359,999261,990891,994783,999647,993378,991831,995401\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 284\n", + "Returned IDs: 996524,993012,990366,997811,991373,997466,998568,993696,999202,992330,990866,996857,993628,992238,993834,997586,993823,997806,995132,992019,998403,992068,995268,995516,999598,994879,990991,992521,997287,994125\n", + "Ground Truth: 996524,993012,990366,997811,991373,997466,998568,993696,999202,992330,990866,996857,993628,992238,993834,997586,993823,997690,997806,995132,992019,998403,992068,995268,995516,999598,994879,990991,992521,997287\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 285\n", + "Returned IDs: 993805,990256,991052,994714,995516,990897,994869,996406,999646,993673,996811,994956,991982,996918,992979,992147,995432,996762,997126,998318,991824,990563,992795,993948,991977,994125,991038,997765,996168,990209\n", + "Ground Truth: 993805,990256,991052,993255,994714,995516,990897,994869,996406,999646,999676,993673,996811,994956,991982,996918,996695,992979,992147,995432,996762,997126,998318,991857,991824,990563,992795,993948,991977,994125\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 286\n", + "Returned IDs: 998779,998862,997640,993412,998999,993293,997204,996727,993831,999026,994382,996436,997319,997732,999021,991923,994409,995401,992169,995216,997355,993280,994436,995374,995940,991743,996042,995274,997871,997832\n", + "Ground Truth: 998779,998862,997640,993412,998999,993293,997204,996727,993831,999026,994382,996436,997319,997732,999021,991923,994409,995401,992169,995216,997355,994436,993280,995374,995940,991743,993226,996042,995274,997871\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 287\n", + "Returned IDs: 994313,990696,995783,999496,999871,990872,996087,997914,991319,990386,998293,999105,997919,997752,997506,996362,998388,990083,997429,993678,999179,994051,993621,990061,995464,993727,993441,996814,993080,996429\n", + "Ground Truth: 994313,990696,995783,999496,999871,990872,996087,997914,991319,990386,998293,999105,997919,997752,997506,996362,998388,990083,997429,993678,999179,994051,993621,992617,990061,995464,993727,993441,993080,996814\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 289\n", + "Returned IDs: 995289,993844,997403,998014,992488,992289,994204,993433,995401,995766,996438,993143,993673,998536,997319,993762,996867,994783,996930,991840,997897,993232,995633,992052,998913,990941,992407,993209,999453,997822\n", + "Ground Truth: 995289,993844,997403,998014,992488,992289,994204,993433,995401,995766,996438,993143,993673,998536,997319,993762,996867,994783,996930,991840,993232,997897,995633,992052,998913,990337,990941,992407,993209,999453\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 293\n", + "Returned IDs: 994672,994247,993715,999139,994577,995801,995349,993504,991552,993055,995839,993139,991245,995737,996884,993181,994743,991648,994351,993609,999079,990885,995622,997076,992747,992052,991332,999806,993489,991605\n", + "Ground Truth: 994672,994247,993715,999139,994577,995801,995349,993504,991552,993055,995839,993139,991245,995737,996884,993181,994743,991648,994351,993609,999079,990885,995622,997076,992747,992052,991332,999806,993489,999927\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 294\n", + "Returned IDs: 995139,995905,995507,999619,993468,996038,991027,995470,996323,993400,997046,999578,997832,995547,992448,997139,993421,995923,997674,999508,990975,996867,994554,999530,990634,994002,993597,997252,996322,994278\n", + "Ground Truth: 995139,995905,995507,999619,993468,992660,996038,991027,995470,996323,993400,997046,999578,997832,995547,997139,992448,993421,995923,997674,999508,990975,996867,994554,999530,990634,994002,993597,997252,996322\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 295\n", + "Returned IDs: 998294,998380,995949,999263,997301,994964,990561,998697,998090,997320,991530,991157,995928,998029,997913,993342,999783,999444,995802,995672,998302,997032,995638,998365,999331,997371,998692,998607,990366,998303\n", + "Ground Truth: 998294,998380,995949,999263,997301,994964,990561,998697,998090,997320,991530,991157,995928,998029,997913,993342,999783,999444,995802,995672,998302,997032,995638,998365,999331,997371,990310,998692,998607,997135\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 296\n", + "Returned IDs: 991720,999343,992147,997842,998921,995372,992169,996915,998259,991743,997192,995515,991889,998877,993080,991485,999698,997918,995927,992944,991406,995443,994280,995771,990648,995336,990432,992309,998229,990917\n", + "Ground Truth: 991720,999343,992147,997842,998921,995372,992169,996915,998259,993857,991743,997192,995515,991889,998877,993080,991485,999698,997918,995927,992944,991406,995443,994088,994280,995771,990648,995336,990432,992309\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 297\n", + "Returned IDs: 999616,995981,991720,995792,997437,999099,999180,990588,999596,992296,993168,998567,997320,994916,995336,997759,997353,998353,995372,999527,992748,993893,998828,991045,997875,990339,996075,998607,995913,995184\n", + "Ground Truth: 999616,995981,991720,995792,997437,999099,999180,990588,999596,992296,993168,998567,997320,994916,990778,995336,997759,997353,998353,995372,999527,992748,994695,993893,998828,991045,997875,990339,996075,998607\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 302\n", + "Returned IDs: 996606,993534,993427,995152,991983,995618,992113,994679,992857,993380,992558,990279,995710,999540,997849,992496,990345,997547,997917,991046,996750,992335,998151,991871,992092,998840,991026,990125,996367,993583\n", + "Ground Truth: 996606,993534,993427,995152,991983,995618,992113,994679,992857,993380,992558,990279,995710,999540,997849,992496,991058,990345,997547,997917,991046,996750,992335,998151,991871,992092,998840,991026,990125,996367\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 303\n", + "Returned IDs: 995459,998142,992824,997775,999171,994679,991432,991817,991370,993706,993077,991550,994276,993583,997864,999908,992392,995503,993647,993828,997906,995983,996219,992280,999639,995737,998061,990765,991923,993260\n", + "Ground Truth: 995459,998142,992824,997775,999171,994679,991432,991817,991370,993706,993077,991550,994276,993583,997864,999908,992392,995503,993647,993828,997906,995983,996219,992280,999639,995737,998061,990765,991923,990538\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 304\n", + "Returned IDs: 999504,997388,997157,996727,990797,991923,991082,998094,997640,999960,994406,999374,997083,995401,998529,997799,997634,998559,999820,990475,998607,993412,990275,991977,994514,998779,997748,998179,999462,995369\n", + "Ground Truth: 999504,997388,997157,996727,990797,991923,991082,998094,997640,999960,994406,999374,997083,995401,998529,997799,997634,998559,999820,990475,998607,995320,993412,990275,991977,994514,998779,997748,998179,999462\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 305\n", + "Returned IDs: 990644,997702,997002,991392,991485,990999,997533,994362,995374,992979,991894,999219,991459,993355,999453,995894,995875,993848,991157,998385,992750,999977,996425,994946,991151,998247,998401,995154,995515,994334\n", + "Ground Truth: 990644,997702,997002,991392,991485,990999,997533,994362,995374,992979,991894,999219,991459,993355,999453,995894,995875,993848,991157,998385,992750,999977,996425,994946,991151,998247,998401,995154,992047,995515\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 306\n", + "Returned IDs: 997475,994891,993611,994282,993821,991153,998065,996535,991066,990301,993046,990479,990961,996728,999775,999832,991597,991111,998279,994280,998320,999205,996743,996495,996431,990774,997919,995600,998221,991273\n", + "Ground Truth: 997475,994891,993611,994282,993821,991153,998065,996535,991066,990301,993046,990479,990961,996728,999775,999832,991597,994316,991111,998279,994280,998320,999205,996743,996495,996431,990774,997919,995600,998221\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 308\n", + "Returned IDs: 993426,990191,992451,991873,999271,999904,995623,995045,997292,997301,997984,996516,998921,998603,993484,990546,995306,995865,993048,992709,990512,998760,991720,997397,996771,998425,999110,992107,991916,999533\n", + "Ground Truth: 993426,990191,992451,991873,999271,999904,995623,995045,997292,997301,997984,996516,998921,998603,993484,990546,995306,995865,993048,992709,990512,998760,991720,997397,996771,998425,999110,992107,992027,991749\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 309\n", + "Returned IDs: 997233,997059,994205,998725,993857,994013,996308,999130,990241,990662,996028,998982,999099,997452,999570,994131,997554,996657,993909,994382,991382,995295,995216,996001,998752,996147,999623,998055,998156,996411\n", + "Ground Truth: 997233,997059,994205,998725,993857,994013,996308,999130,990241,990662,996028,995600,998982,999099,997452,999570,994131,997554,996657,993909,994382,991382,995295,995216,996001,998752,999623,996147,998055,998156\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 311\n", + "Returned IDs: 992041,992124,999639,997258,992238,990453,992497,993696,998758,998260,992167,992225,991984,995516,993834,992927,994809,993283,990949,997370,997320,999041,998950,991157,992521,991459,996500,995899,996075,992896\n", + "Ground Truth: 992041,992124,999639,997258,992238,990453,992497,993696,998758,998260,992167,992225,991984,995516,993834,992927,994809,993283,990949,997370,997320,999041,998950,991157,992521,991459,990862,996500,995899,996075\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 312\n", + "Returned IDs: 990821,990310,991840,995895,991157,990271,993717,996809,991015,995633,990941,993391,997331,994922,993642,993143,998033,998950,996840,994830,991848,993342,999453,999784,996328,994742,999122,992169,992612,997822\n", + "Ground Truth: 990821,990310,991840,995895,991157,990271,993717,996809,991015,995633,990941,993391,997331,994922,993642,993143,996408,998033,998950,996840,994830,991848,993342,999453,999784,996328,994742,999122,992169,992612\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 315\n", + "Returned IDs: 992771,997395,993566,990194,993106,996393,999994,993931,999673,995895,992685,991188,996408,990951,999543,995620,999555,991037,999430,990407,993224,998536,998006,996762,990796,990434,993011,998383,996395,998855\n", + "Ground Truth: 992771,997395,993566,990194,993106,996393,999994,993931,999673,995895,992685,991188,996408,990951,999543,995620,999555,991037,999430,990407,993224,998536,998006,996762,990796,990434,993011,995211,998383,995401\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 318\n", + "Returned IDs: 991144,993719,990257,993917,992636,993715,996986,990438,990951,997168,999334,996357,993150,993637,995991,998076,998716,998561,990089,997879,998348,996408,997790,996328,994522,994834,991081,997669,993139,996201\n", + "Ground Truth: 991144,993719,990257,993917,992636,993715,996986,990438,990951,997168,999334,996357,993150,993637,995991,998076,998716,998561,990089,997879,998348,996408,997790,996328,995002,994522,994834,991081,997669,993139\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 319\n", + "Returned IDs: 992747,998303,993489,991037,992571,990350,998401,999840,990409,991332,995536,990930,996217,994468,997002,997977,992082,995894,996062,993504,994200,995203,993260,990885,990096,991530,996677,997368,995025,998007\n", + "Ground Truth: 992747,998303,993489,991037,992571,990350,998401,999840,990409,991332,995536,990930,996217,994468,997002,997977,992082,995894,996062,993504,994200,995203,993260,990885,990096,991530,996677,997368,995025,995823\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 320\n", + "Returned IDs: 993809,998080,998495,995401,996371,992571,993255,991052,996370,992318,990083,998950,993696,992847,993515,997723,995932,990532,993478,999701,998221,997586,992944,996805,997184,991412,993444,998416,998968,990776\n", + "Ground Truth: 993809,998080,998495,995401,996371,992571,993255,991052,996370,992318,990083,998950,993696,992847,993515,997723,992753,995932,990532,993478,999701,998221,997586,992944,996805,997184,991412,993444,990711,998416\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 321\n", + "Returned IDs: 990999,998990,998179,995401,997258,991459,997191,996379,996198,996727,993561,990453,997799,990354,993848,999219,992684,991392,991630,999021,994946,993412,994319,996247,999639,990838,991792,995994,990561,992979\n", + "Ground Truth: 990999,998990,998179,995401,997258,991459,997191,996379,996198,996727,993561,990453,997799,990354,993848,999219,992684,991392,991630,999021,994946,993412,994319,996247,999639,990838,991792,999228,995994,990561\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 322\n", + "Returned IDs: 990866,997379,997586,993708,996397,993637,996429,993552,999308,999574,999647,999500,995701,997013,992068,990547,991157,997811,992944,991808,999216,992522,992047,997555,999598,997496,998180,992682,995455,999473\n", + "Ground Truth: 990866,997379,997586,993708,996397,993637,996429,993552,999308,999574,999647,999500,995701,997013,992068,990547,991157,997811,992944,991808,999216,992522,992047,997555,999598,997496,998180,998790,992682,995455\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 323\n", + "Returned IDs: 993160,995062,998080,993971,999961,992150,997634,998806,997061,997395,998316,998650,992677,990131,993255,993263,994576,990789,995886,996273,990103,992604,994774,996139,993122,992493,998528,991360,992558,992148\n", + "Ground Truth: 993160,995062,998080,993971,999961,992150,997634,998806,997061,997395,998316,998650,992677,997429,990131,993255,993263,994576,990789,995886,996273,990103,992161,992604,994774,996139,993122,992493,998528,991360\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 324\n", + "Returned IDs: 991825,996028,994689,995823,998140,998825,993378,997051,995649,990811,991426,997373,994012,991840,998982,993906,992201,997417,993890,998382,994307,997541,995089,993725,991989,992780,999446,999740,995680,999286\n", + "Ground Truth: 991825,996028,994689,995823,998140,998825,993378,997051,995649,990811,991426,997373,994012,991840,998982,993906,992201,997417,993890,998382,994307,995502,997541,995089,993725,991989,992780,999446,999740,995680\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 325\n", + "Returned IDs: 994630,996961,997525,999761,994699,993518,991895,996732,997913,998246,999710,992546,992709,994939,994932,995949,994757,998744,996785,997599,998332,990397,992502,997535,990925,990509,991873,992119,991479,996890\n", + "Ground Truth: 994630,996961,997525,999761,994699,993518,991895,996732,997913,998246,999710,992546,992709,994939,998016,994932,995949,994757,998744,997599,996785,998332,990397,992502,997535,990925,990509,991873,992119,991479\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 326\n", + "Returned IDs: 997976,998170,994894,997479,999182,994757,991782,997320,999144,997918,993673,995981,995279,995886,990231,994100,991379,990660,997571,998011,995336,990958,994819,994334,997192,999527,991181,990647,990588,997777\n", + "Ground Truth: 997976,998170,994894,997479,999182,994757,991782,997320,999144,997918,993673,995981,995279,995886,990231,994100,991379,990660,997571,998011,995336,990958,998163,994819,994334,997192,999527,991181,990647,990588\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 327\n", + "Returned IDs: 995274,996411,990293,996657,993412,996001,993909,995216,993003,998302,999026,998779,996810,998952,990974,991995,990845,996759,998862,995289,998058,999835,999587,990957,992792,999099,997529,996219,999343,993168\n", + "Ground Truth: 995274,996411,990293,996657,993412,996001,993909,995216,993003,998302,999026,998779,996810,998952,990974,991995,990845,996759,998862,995289,992120,995899,998058,999835,999587,990957,992792,997969,999099,997529\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 328\n", + "Returned IDs: 998257,995373,991204,997110,990657,993115,997568,994643,998858,994454,997030,997589,999129,993795,991987,991386,995401,993000,996393,997675,997683,992762,999136,998536,991052,990096,992861,995453,997918,990005\n", + "Ground Truth: 998257,995373,991204,997110,990657,993115,997568,994643,998858,994454,997030,997589,999129,993795,991987,999530,991386,995401,993000,996393,997675,997683,992762,999136,998536,991052,990096,992861,998697,995453\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 331\n", + "Returned IDs: 990154,998006,995815,999555,996728,994181,997666,992771,995835,999673,990952,990006,999543,997884,992282,990951,994834,999388,994467,998861,999663,996505,995655,991596,994103,990429,998067,997358,995536,994756\n", + "Ground Truth: 990154,998006,995815,999555,996728,994181,997666,992771,995835,999673,990952,990006,999543,997884,992282,990951,994834,999388,994467,998861,999350,999663,996505,995655,991596,994103,990429,998067,997358,995536\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 334\n", + "Returned IDs: 991377,992894,994419,995289,997386,997822,993412,995702,995923,991923,993511,996080,991603,990847,993561,996610,996867,991958,996360,997529,996745,993433,994925,996322,995200,994409,992130,995401,997937,999782\n", + "Ground Truth: 991377,992894,994419,995289,997386,997822,993412,995702,995923,991923,993511,996080,991603,990847,993561,996610,996867,991958,996360,997529,996745,996147,993433,994925,996322,995200,996127,994409,992130,997920\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 337\n", + "Returned IDs: 996531,998397,993747,990956,996276,999514,991747,998293,996867,998478,992703,994119,990466,994541,996232,998486,992160,990116,997822,994444,999264,999646,990103,993727,996294,997420,995159,992992,999244,991145\n", + "Ground Truth: 996531,998397,993747,990956,996276,999514,991747,998293,996867,998478,992703,994119,990466,994541,998165,996232,998486,992160,990116,997822,994444,999264,999646,990103,993727,996294,997420,995159,992992,999244\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 342\n", + "Returned IDs: 993364,992126,991994,993885,991792,990772,997876,990557,993820,995841,994162,991212,991498,995372,997320,999982,998444,994673,992788,995515,999453,996489,997973,999819,997002,996001,990724,996959,993032,994081\n", + "Ground Truth: 993364,992126,991994,993885,991792,990772,997876,997380,990557,993820,995841,994162,991212,991498,995372,997320,999982,995502,998444,994673,992788,995515,999453,996489,997973,999819,997002,996001,990724,996959\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 344\n", + "Returned IDs: 994277,993721,994233,994416,991409,996685,995401,997403,995899,992944,996024,997320,998950,991348,991416,998921,996210,996075,994081,997448,990364,992318,995372,998913,996178,991650,991997,991743,997305,999159\n", + "Ground Truth: 994277,993721,994233,994416,991409,996685,995401,997403,995899,992944,996024,997320,998950,991348,991416,998921,996210,993121,996075,994081,997448,990364,995372,992318,998913,996178,991650,991997,991743,997305\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 345\n", + "Returned IDs: 999827,996001,997320,994964,991498,993080,997920,999453,994081,995841,992182,999099,994478,992395,993168,995216,997860,998750,993280,997918,995274,990146,993477,993289,990772,993648,995102,993484,991212,990532\n", + "Ground Truth: 999827,996001,997320,994964,991498,993080,993454,997920,999453,994081,995841,992182,999099,999026,994478,992395,993168,995216,997860,998750,993280,997918,995274,990146,993477,993289,990772,993648,995102,991007\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 346\n", + "Returned IDs: 992907,995588,997465,998582,993091,999676,996840,995515,993028,993622,999076,991840,990300,993355,990310,994922,992979,999073,996352,991999,996686,993781,999077,994612,990490,990432,993688,991160,992218,999620\n", + "Ground Truth: 992907,995588,997465,998582,993091,999676,996840,995515,993028,993622,999076,991840,990300,993355,990310,994922,992979,999073,996352,999604,991999,996686,993781,999077,994612,990490,990432,993688,991160,992218\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 347\n", + "Returned IDs: 997909,997299,998890,996256,999925,993003,994409,995401,997918,993537,991357,991743,991795,992763,996815,996609,995864,994106,990087,990625,994062,999926,996613,997218,994865,996232,998205,991892,996576,997612\n", + "Ground Truth: 997909,997299,998890,996256,999925,993003,994409,995401,997918,993537,991357,991743,991795,992763,996815,996609,995864,997555,994106,990087,990625,994062,990432,999926,996613,997218,994865,996232,998205,991892\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 349\n", + "Returned IDs: 993754,997781,992604,995527,995807,999125,994774,999445,995105,999662,990915,998059,991051,999478,996691,992387,990131,997212,990729,997462,997693,997110,993632,998528,992855,995625,994539,995953,996206,996236\n", + "Ground Truth: 993754,997781,992604,995527,995807,999125,994774,999445,995105,999662,990915,998059,991051,999478,996691,992387,990131,998297,997212,990729,997462,997693,997110,993632,998528,992855,995625,994539,995953,996206\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 352\n", + "Returned IDs: 991498,993484,994503,997597,999005,999263,995372,997320,997301,995841,994917,996560,992637,998849,998966,998634,991743,990829,990936,995034,994183,998779,998839,990585,998445,992510,999658,997305,994611,998163\n", + "Ground Truth: 991498,993484,994503,997597,999005,999263,995372,997320,997301,995841,994917,996560,992637,998849,998966,998634,991743,990829,990936,995034,994183,998779,998839,990585,998445,992510,999658,997305,997908,994611\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 356\n", + "Returned IDs: 999587,992884,993289,999453,996657,997554,996001,990845,994531,997002,990399,996810,992979,996028,994946,995401,996147,993032,996368,995216,992521,995281,996759,999827,994809,993280,992558,990956,998495,990255\n", + "Ground Truth: 999587,992884,993289,999453,996657,997554,996001,990845,994531,997002,990399,996810,992979,996028,994946,995401,996147,993032,996368,995216,992521,995281,996759,997023,999827,994809,993280,992558,990956,998495\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 357\n", + "Returned IDs: 991063,993809,998225,998302,994879,999444,997906,995638,990866,992748,996805,998950,997913,998123,996230,996075,999977,996811,995672,995755,997320,999831,996001,998580,995701,998697,992394,990096,990083,996761\n", + "Ground Truth: 991063,993809,998225,998302,994879,999444,997906,995638,990866,992748,996805,998950,997913,998123,996230,996075,991927,999977,996811,995672,995755,997320,999831,993821,996001,998580,995701,998697,991311,992394\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 358\n", + "Returned IDs: 995580,991677,995775,999333,995266,999301,992668,999414,996499,994464,996192,998170,992134,997913,998216,993222,992946,997818,997192,998152,990957,991422,995014,998686,990647,998600,997106,995870,997841,994748\n", + "Ground Truth: 995580,991677,995775,999333,995266,999301,992668,999414,996499,998599,994464,996192,998170,992134,997913,998216,993222,992946,994904,997818,997192,991081,998152,990957,991422,995014,998686,990647,998600,997106\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 359\n", + "Returned IDs: 990323,991037,999748,991596,995915,995385,990194,999576,995712,996702,997242,994955,995562,990148,994200,995349,990096,997666,993179,994231,999435,998583,993866,992571,993458,999384,995655,990596,996424,990876\n", + "Ground Truth: 990323,991037,999748,991596,995915,995385,990194,999576,995712,996702,997242,994955,995562,994020,990148,994200,995349,990096,997666,993179,994231,999435,998583,993866,992571,993458,999384,995655,990596,996424\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 360\n", + "Returned IDs: 995559,993695,991179,993038,995560,995946,998790,999898,996210,990435,991923,997920,992944,992047,998416,996042,995107,997586,993055,996370,990096,995720,995332,999430,994815,993241,998349,998950,993324,999806\n", + "Ground Truth: 995559,993695,991179,993038,995560,995946,998790,999898,996210,990435,991923,997920,992944,992047,998416,996042,995107,997586,993055,996370,990096,995720,995332,999430,994815,995401,993241,998349,998950,993324\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 361\n", + "Returned IDs: 995891,991544,998651,995946,993561,994413,996472,990096,991958,998973,993566,992448,992260,991247,997252,997386,998545,998451,994285,995536,991447,994365,995423,995303,992631,995146,993433,999543,994409,992801\n", + "Ground Truth: 995891,991544,998651,995946,993561,994413,996472,990096,991958,998973,993738,993566,992448,992260,991247,997252,997386,998545,998451,994285,995536,991447,994365,995423,995303,992631,995146,993433,999543,999673\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 362\n", + "Returned IDs: 994589,991666,995401,992933,997723,991630,996659,990517,998530,999977,995701,996379,996864,998350,998874,991603,998990,999461,992367,998045,999000,998536,994177,995994,994409,992977,992790,992521,994946,999228\n", + "Ground Truth: 994589,991666,995401,992933,997723,991630,996659,990517,998530,999977,995701,996379,996864,998350,998874,991603,999461,998990,992367,998045,999000,998536,994177,995994,994409,992977,992790,991151,992521,994946\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 366\n", + "Returned IDs: 995633,999676,997374,994317,992289,997319,995401,994529,995289,990903,991743,996867,990126,995268,998221,995274,990432,993402,997723,994878,999453,994409,999587,993920,995020,998913,994850,994343,990435,999533\n", + "Ground Truth: 995633,999676,997374,994317,992289,997319,995401,994529,995289,990903,991743,996867,990126,993412,995268,998221,995274,990432,993402,997723,994878,999453,994409,999587,993920,995020,998913,994850,994343,990435\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 367\n", + "Returned IDs: 994964,999151,990545,996219,995961,999331,991345,999261,994843,999352,999367,990922,994896,991921,994577,999210,991382,993238,993877,997720,993215,994705,995927,991378,994782,992446,990210,995739,994850,991751\n", + "Ground Truth: 994964,999151,990545,996219,995961,999331,991345,999261,994843,999352,999367,990922,994896,991921,994577,999210,991382,993238,992942,993877,998294,993154,997720,993215,994705,995927,991378,994782,990210,992446\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 368\n", + "Returned IDs: 995166,993857,998982,994689,991075,997208,996028,992374,991585,995456,998427,993725,998182,990328,994307,995823,990097,993053,994765,995633,992480,996506,993906,995295,997309,995289,990811,992558,998180,991315\n", + "Ground Truth: 995166,993857,998982,994689,991075,997208,996028,992374,991585,995456,998427,993725,998182,990328,994307,995823,990097,993053,995401,994765,995633,992480,996506,993906,995295,997309,995289,990811,992558,998180\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 370\n", + "Returned IDs: 997320,993484,997913,994604,996143,993154,998967,999948,995802,992442,990066,997470,998170,999658,990532,992119,995372,992330,990949,995865,999851,992546,993157,996075,995942,998808,992896,999535,995351,990585\n", + "Ground Truth: 997320,993484,997913,994604,996143,993154,998967,999948,995802,995228,992442,990066,997470,998170,999658,990532,992119,995372,992330,990949,995865,999851,992546,993157,996075,995942,998808,992896,999535,995351\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 371\n", + "Returned IDs: 991975,993641,990429,995168,992110,999176,995231,993629,998858,993381,995780,997078,991455,996623,991332,992365,991385,993560,999555,991987,996571,994413,990976,994834,992926,990449,993191,998149,991038,995655\n", + "Ground Truth: 991975,993641,990429,995168,992110,999176,995231,993629,998858,993381,995780,997078,991455,996623,991332,992365,991385,993560,999555,991987,996571,994413,990976,994834,992926,997546,990449,998731,993191,998149\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 372\n", + "Returned IDs: 990249,993811,992806,995967,993727,997195,992121,999240,999479,993121,993583,990121,996750,993473,992558,994163,999400,998493,998820,991447,997918,993158,993911,992864,990117,995626,997124,994168,990395,999098\n", + "Ground Truth: 990249,993811,992806,995967,993727,997195,992121,999240,999479,993121,993583,990121,996750,993473,992558,999887,994163,999400,997897,998493,992407,998820,991447,997918,998559,993158,993911,992864,990117,995626\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 376\n", + "Returned IDs: 995372,998849,992944,993241,990585,990060,995375,996809,990489,999544,998567,990226,995560,995841,999711,991498,999210,996318,997320,994972,994402,991994,997848,996001,990043,995641,990894,997436,997113,995443\n", + "Ground Truth: 995372,998849,992944,993241,990585,990060,995375,996809,990489,999544,998567,990226,995560,995841,999711,991498,999210,996318,997320,995401,994972,993533,994402,991994,997848,993388,996001,990043,995641,990894\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 379\n", + "Returned IDs: 992532,995974,998686,998334,991480,999451,995560,992349,992645,996894,999711,993976,995314,998417,998079,992654,992383,997521,990383,996196,990401,996262,991015,991208,990376,997758,998061,999639,999913,991977\n", + "Ground Truth: 992532,995974,998686,998334,991480,999451,995560,992349,992645,996894,993174,999711,993976,995314,998417,998079,992654,992383,997521,990383,996196,990401,998849,996262,991015,999216,991208,990376,997758,998061\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 381\n", + "Returned IDs: 995482,999179,990157,994806,997997,993466,995106,991664,995055,998256,996633,993731,996283,991061,994166,997411,993820,998748,996386,995778,995777,996328,993111,998543,992563,998221,999759,990806,992364,990987\n", + "Ground Truth: 995482,999179,990157,994806,997997,993466,995106,991664,995055,998256,996633,993731,996283,991061,990083,994166,997411,993820,998748,996386,995778,995777,996328,993111,998543,992563,998221,999759,990806,992364\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 382\n", + "Returned IDs: 996147,992161,990771,995167,996001,997790,997918,999898,991667,992950,993289,996657,997554,998215,999150,990399,999099,991311,999453,991212,994266,995855,997163,991585,992384,996549,995281,990314,993168,998720\n", + "Ground Truth: 996147,992161,990771,995167,996001,997790,997918,999898,991667,992950,993289,996657,997554,998215,999150,990399,999099,991311,999453,991212,994266,995855,993143,997163,991585,991485,992384,996549,995281,990314\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 383\n", + "Returned IDs: 992558,990538,998950,990279,999639,996196,997775,997918,997258,994163,992615,995552,999206,999860,996609,990273,997859,999564,996592,998411,991522,994280,995340,994742,995372,999715,992436,991498,991791,993077\n", + "Ground Truth: 992558,990538,998950,990279,999639,996196,997775,997918,995693,997258,994163,992615,995552,999206,999860,996609,990273,997859,999564,996592,998411,991522,994280,995340,994742,995372,999715,992436,991498,991791\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 385\n", + "Returned IDs: 997852,997476,999077,999676,992169,994922,991334,990490,996325,992612,999916,990432,993260,997822,997496,993717,998041,993673,991840,991743,994964,992813,999453,998950,999843,997135,993355,994742,999533,994318\n", + "Ground Truth: 997852,997476,999077,999676,992169,994922,991334,990490,996325,997204,992612,999916,990432,993260,997822,997496,993717,998041,993673,991840,991743,994964,992813,999453,998950,994044,999843,997135,993355,990214\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 389\n", + "Returned IDs: 990742,991593,991662,991016,997826,998899,999701,997913,995748,991502,998003,999374,991616,997320,995928,998170,993238,994964,991479,993885,999948,993606,998294,992978,991743,990816,999262,994735,995917,992255\n", + "Ground Truth: 990742,991593,991662,991016,997826,998899,999701,997913,995748,991502,998003,999374,991616,997320,999390,995928,990902,991237,998170,993238,994964,991479,993885,999948,993606,998294,992978,991743,990816,998663\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 390\n", + "Returned IDs: 990249,996609,998820,991248,995124,992558,999240,996762,991350,994457,991266,995289,995377,997822,996606,993412,995332,995354,997723,999646,995401,995119,997498,991330,991449,998874,999898,992576,998529,996750\n", + "Ground Truth: 990249,996609,998820,991248,995124,992558,993433,999240,996762,991350,994457,991266,995289,995377,997822,996606,993412,995332,996294,992469,995354,997723,999646,995401,995119,997498,991330,991449,998874,999898\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 393\n", + "Returned IDs: 990487,996451,999774,999191,996194,997770,991977,994254,997251,996042,999179,990102,998968,997115,990678,992558,991447,992979,996658,994387,992086,998104,991303,990705,994280,993275,991743,997918,997529,991061\n", + "Ground Truth: 990487,996451,999774,999191,996194,991720,998528,999343,997770,991977,994254,997251,996042,999179,990102,998968,997115,990678,992558,991447,992979,996658,994387,992086,998104,991303,990705,994280,993275,991743\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 394\n", + "Returned IDs: 990535,999144,994217,993168,992630,998951,993083,997015,992331,995336,997320,995947,999437,997082,994964,990925,992748,993389,995252,997002,996001,999019,993484,994757,994916,995824,994854,998090,990958,992047\n", + "Ground Truth: 990535,999144,994217,993168,992630,998951,993083,997015,992331,995336,999916,997320,995947,999437,997082,994964,990925,992748,993389,995252,997002,995216,996001,999019,993484,994757,994916,995824,994854,998090\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 395\n", + "Returned IDs: 994280,991334,997023,995968,990214,995377,995401,999701,991923,994835,995820,991743,990921,995886,994655,992165,992582,995824,994982,992521,998302,992384,993561,997918,991269,994496,996762,995336,998295,995211\n", + "Ground Truth: 994280,991334,997023,995968,990214,995377,995401,999701,991923,994835,995820,991743,998374,990921,995886,994655,992165,992582,995824,994982,992521,998302,992384,993561,997918,991269,994496,996762,995336,998295\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 399\n", + "Returned IDs: 998908,998921,994244,997539,997301,999995,993581,995154,995910,993143,999219,998539,990760,996014,995554,991157,993364,993662,997777,997370,990561,992124,999358,996040,998029,998726,997371,993250,996092,998894\n", + "Ground Truth: 998908,998921,994244,997539,997301,999995,993581,995154,990702,999426,995910,993143,999219,992461,998539,990760,996014,995554,991157,993900,993364,997158,993662,997777,997370,990561,992124,999358,996040,993005\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 402\n", + "Returned IDs: 997643,994724,997210,992638,995044,997683,990657,997589,999156,999117,998918,992206,992623,993076,998257,996393,998277,993795,994660,993666,991797,999808,995207,991309,990167,998787,991204,990361,998383,992760\n", + "Ground Truth: 997643,994724,997210,992638,995044,997683,990657,997589,999156,999117,998918,992206,992623,993076,998257,996393,998277,993795,994660,993666,991797,999808,995207,991309,990167,998787,991204,994917,990361,998383\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 403\n", + "Returned IDs: 998149,990658,994869,999663,991038,999834,993055,994155,999736,997506,999139,998858,995780,994467,995237,991169,999555,992421,998831,993629,995516,996406,999896,996623,999806,995755,998383,998720,991154,996446\n", + "Ground Truth: 998149,990658,994869,999663,991038,999834,993055,994155,999736,997506,999139,998858,995780,994467,995237,991169,999555,992421,998831,993629,995516,996406,999896,996623,999806,995755,998383,998720,991154,991835\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 406\n", + "Returned IDs: 994577,992068,998816,991015,999806,999194,997379,995039,999122,993628,999432,992423,993012,991157,991987,996281,998568,998950,998858,993143,997466,990991,998437,998257,999267,991713,999568,992330,990479,991844\n", + "Ground Truth: 994577,992068,998816,991015,999806,999194,997379,995039,999122,993628,995453,999432,992423,993012,991157,991987,996281,998568,998950,998858,993143,997466,990991,998437,998257,999267,991713,999568,992330,990479\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 407\n", + "Returned IDs: 994717,994809,990255,990361,997088,995401,999587,993515,994092,992293,996657,999629,998593,996043,996809,995701,996759,990573,994431,996930,995289,992558,993848,992367,998536,997897,998990,996393,998250,996777\n", + "Ground Truth: 994717,994809,990255,990361,997088,995401,999587,993515,994092,992293,996657,999629,998593,996043,996809,995701,996759,990573,994431,996930,995289,992558,993848,997411,992367,998536,997897,998990,996393,998250\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 409\n", + "Returned IDs: 996690,993363,996963,991772,991157,991550,993370,993411,993706,999393,994163,996196,999639,997775,998697,990159,996926,995013,992349,990279,992770,999908,992423,992738,997225,997258,998142,995599,999041,995804\n", + "Ground Truth: 996690,993363,996963,991772,991157,991550,993370,993411,993706,999393,992383,994163,996196,999639,997775,998697,990159,996926,995013,992349,990279,992770,991706,999908,992423,992738,997225,997258,998142,995599\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 410\n", + "Returned IDs: 998170,994240,997699,993431,996592,994584,999077,992880,997470,993260,997320,997918,991801,995094,991348,992927,994666,999701,996296,998180,994450,995826,991928,993696,995266,995357,991564,991131,994964,992748\n", + "Ground Truth: 998170,994240,997699,993431,996592,994584,999077,992880,997470,993260,997320,997918,991801,995094,991348,993454,992927,994666,999701,996296,998180,994450,995826,991928,993696,995266,995357,991564,991131,994964\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 411\n", + "Returned IDs: 995266,991593,992069,998899,994399,997320,991774,992255,991580,990957,998967,991662,999290,997470,997301,999077,993662,995826,991479,993852,998090,999602,994604,990910,994290,996296,992978,990650,993034,993402\n", + "Ground Truth: 995266,991593,992069,998899,994399,997320,991774,992255,991580,990957,998967,991662,999290,997470,997301,999077,993662,995826,991479,993852,998090,999602,994604,990910,994290,996296,992978,992946,990650,998801\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 412\n", + "Returned IDs: 992521,997545,993269,997466,996552,991878,991969,993107,992097,994353,998238,993438,991555,993353,998075,998910,998346,995193,994508,991760,997216,995813,998205,998568,992448,994636,994954,993738,997723,995864\n", + "Ground Truth: 992521,997545,993269,997466,996552,991878,991969,993107,992097,994353,998238,993438,991555,998564,993353,998075,998910,998346,995193,994508,991760,997216,995813,998205,998272,995644,998568,992448,994636,994954\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 413\n", + "Returned IDs: 990956,992558,996809,994742,995963,991052,999179,992052,990776,992604,997723,990361,998495,997752,994643,995408,994351,993948,990657,998543,993799,995401,993079,991319,995105,992685,997914,993121,996043,993820\n", + "Ground Truth: 990956,992558,996809,994742,995963,991052,999179,992052,990776,992604,997723,990361,998495,997752,994643,993077,995408,994351,993948,990657,998543,991257,993799,995401,993079,991319,995105,992685,997914,993121\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 417\n", + "Returned IDs: 991198,997789,990043,999620,998232,991196,996681,998236,997275,998983,993582,990155,991801,990858,990809,993570,998837,996318,994922,994785,992907,999076,993675,991510,991999,994964,990585,997876,993237,999053\n", + "Ground Truth: 991198,997789,990043,999620,998232,991196,996681,998236,997275,998983,993582,990155,991801,990858,990809,993570,998837,996318,994922,994785,992907,999076,993675,991510,991999,994964,990585,997876,998280,993237\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 420\n", + "Returned IDs: 998886,992293,994382,990662,997868,990361,998017,990399,996809,999922,997466,995470,992582,998495,993390,992862,991603,999587,996042,993831,994793,997723,994689,995679,997529,991578,998982,995562,994730,990638\n", + "Ground Truth: 998886,992293,994382,990662,997868,990361,998017,990399,996809,999922,997466,995470,992582,998495,993390,992862,991603,999587,996147,996042,993831,994793,997723,994689,995679,997529,991578,998982,995562,994730\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 423\n", + "Returned IDs: 998470,995214,992558,999791,999767,998122,995222,998279,994972,998495,997732,990956,998765,996001,997603,999191,999827,992713,995438,990091,990089,996219,996647,998710,990364,994280,999138,995107,999852,994119\n", + "Ground Truth: 998470,995214,992558,999791,999767,998122,995222,998279,994972,998495,997732,990956,998765,996001,997603,999191,999827,992713,995438,990091,997920,990089,999916,996219,996647,998710,990364,994280,999138,995107\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 424\n", + "Returned IDs: 990811,998593,996147,999173,991000,996657,990130,992326,993483,996028,998878,990771,992727,996001,994838,998934,995158,999740,995864,990573,998250,993391,992289,993813,995289,995200,995182,993848,996302,997007\n", + "Ground Truth: 990811,998593,996147,999173,991000,996657,990130,992326,993483,996028,998878,990771,992727,996001,998934,994838,995158,999740,995864,990573,998250,993391,996864,992289,993813,995289,995200,995182,993848,996302\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 425\n", + "Returned IDs: 990382,999543,994510,993515,993931,999139,990096,998593,998250,996062,991685,995889,994463,991596,995536,998495,997790,995895,990930,996733,994191,991011,998376,996001,996549,999994,995017,993778,999091,991066\n", + "Ground Truth: 990382,999543,994510,993515,993931,999139,990096,998593,998250,996062,991685,995889,994463,991596,995536,998495,994796,997790,995895,990930,996733,994191,991011,998376,996001,996549,999994,995017,993778,999091\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 427\n", + "Returned IDs: 992862,997586,995005,991490,995701,990786,999806,993809,995536,998708,999653,995455,996082,993637,993470,999444,990167,998200,997506,993834,990271,999087,993515,998697,993504,995203,998536,991038,993055,995672\n", + "Ground Truth: 992862,997586,995005,991490,995701,990786,999806,993809,995536,998708,999653,995455,996082,993637,993470,999444,990167,998200,997506,993834,990271,996370,999087,993515,998697,993504,995203,993821,998536,997577\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 429\n", + "Returned IDs: 996734,995401,990337,997191,998297,995685,993412,995373,998045,991603,994409,993561,998999,991052,997406,996968,992448,995940,996277,993288,991140,997723,994948,990930,997918,995648,999530,997822,992685,995994\n", + "Ground Truth: 996734,995401,990337,997191,998297,995685,993412,995373,998045,991603,994409,993561,998999,991052,997406,996968,992448,995940,996277,993288,991140,997723,994948,990930,997918,995648,999530,997822,995873,992685\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 430\n", + "Returned IDs: 994956,996219,996328,990214,999179,996811,997442,999500,994387,996149,993727,995332,998256,998968,995438,997420,999560,994783,995562,995130,997002,994233,995895,999540,999352,990039,994206,992237,994452,997945\n", + "Ground Truth: 994956,996219,996328,990214,999179,996811,997442,999500,994387,996149,993727,995332,998256,998968,995438,991212,997420,999560,994783,995562,996071,995130,997002,994233,995895,999352,999540,990039,994206,992237\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 431\n", + "Returned IDs: 994809,997218,995401,992790,990841,991671,996762,991197,991334,996930,993673,992749,992558,990312,992636,991459,994835,995864,992685,993003,992448,998205,996162,998104,996187,990181,993275,995873,997023,996001\n", + "Ground Truth: 994809,997218,995401,992790,990841,991671,996762,991197,991334,996930,993673,992749,992558,990312,992636,991459,994835,995864,992685,998855,993003,992448,998205,996162,998104,996187,990181,993275,995873,996438\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 432\n", + "Returned IDs: 996279,994533,999301,996192,994688,994757,993852,999883,990215,990469,994854,990275,998190,996714,999089,992511,993457,999327,995034,991379,997012,999500,992384,998170,992087,999548,998011,999953,992854,997605\n", + "Ground Truth: 996279,994533,999301,996192,994688,994757,993852,999883,990215,990469,994854,990275,998190,996714,999089,992511,993457,999327,995034,991379,997012,999500,992384,998170,992087,991650,999548,998011,999953,992854\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 433\n", + "Returned IDs: 999820,990561,995327,992979,994906,997910,992790,994946,994673,993289,994819,990304,995968,998011,991074,996425,994982,995401,996714,994170,997218,999715,994334,997396,994188,997002,994922,998210,999977,991607\n", + "Ground Truth: 999820,990561,995327,992979,994906,997910,992790,994946,994673,993289,994819,990304,995968,998011,992047,991074,996425,994982,995401,996714,994170,997218,999715,994334,997396,994188,997002,994922,998210,999977\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 434\n", + "Returned IDs: 992950,997320,996904,993662,998540,999109,992451,998302,997470,993237,994964,997913,997379,992443,996890,993791,998550,999710,999210,998837,991626,993796,994192,992069,991801,993018,991582,997135,992733,998950\n", + "Ground Truth: 992950,997320,996904,993662,998540,999109,992451,998302,997470,993237,994964,997913,997379,992443,992140,996890,993791,998550,999710,999210,998837,991626,993796,994192,990615,992069,991801,993018,991582,997135\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 437\n", + "Returned IDs: 994749,999544,994826,997249,995968,997498,999254,994081,997888,995372,990772,990043,995932,991440,993071,999676,996309,996506,991787,994171,992531,991880,999453,995841,997661,998734,994687,997189,995250,991002\n", + "Ground Truth: 994749,999544,994826,997249,995968,997498,999254,994081,997888,995372,990772,990043,995932,991440,993071,999676,996309,996506,991787,998654,994171,992531,991880,999453,995841,997661,996785,998734,994240,994687\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 439\n", + "Returned IDs: 998536,992110,991644,990449,991620,993001,992282,992595,990399,992832,999827,997506,990154,998067,995655,990942,995326,991667,996328,992444,998625,992318,996408,999994,999555,992862,993993,995030,994380,997494\n", + "Ground Truth: 998536,992110,991644,990449,991620,993001,992282,992595,990399,992832,999827,997506,990154,998067,995655,990942,995326,997723,991667,996328,992444,998625,998861,992318,996408,999994,999555,992862,993993,995030\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 440\n", + "Returned IDs: 999594,992451,993419,990936,992583,990089,998163,999516,993060,995659,999791,996279,996755,993848,994776,992546,995792,992470,999757,993630,993484,996001,996516,993490,992785,997320,998921,996075,995454,999900\n", + "Ground Truth: 999594,992451,993419,990936,992583,990089,998163,999516,993060,994325,995659,999791,996279,990862,996755,993848,994776,992546,994199,995792,992470,999757,993630,993484,996001,996516,993490,992785,997320,998921\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 441\n", + "Returned IDs: 991178,990980,993914,990648,999343,996219,997984,994266,999404,993238,994964,998500,992546,995942,990146,999824,993072,990595,994415,990650,990736,996028,993518,994889,998458,990545,998581,997855,991325,998921\n", + "Ground Truth: 991178,990980,993914,990648,999343,996219,997984,994266,999404,993238,994964,998500,992546,995942,990146,993072,999824,990595,994415,993048,990650,990736,996028,993518,994889,998458,990545,998581,997855,991325\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 442\n", + "Returned IDs: 991856,993960,991202,994497,994604,999260,990224,995135,999633,996854,990958,999374,999793,991889,998190,994922,995947,995259,990980,991593,998921,995365,993640,994266,990660,995443,994195,998500,997192,991196\n", + "Ground Truth: 991856,993960,991202,994497,994604,992323,999260,999261,990224,995135,999633,996854,991733,990958,999374,999793,997470,991889,999398,998190,994922,995947,995259,990980,991593,995959,998921,995365,993640,993570\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 447\n", + "Returned IDs: 990366,990089,995569,994373,999898,993401,990271,996219,997855,995306,994919,999417,999610,994279,994792,999651,992174,995095,995096,995210,990438,995551,991593,991172,999192,999791,995868,993402,990173,991751\n", + "Ground Truth: 990366,990089,995569,994373,999898,993401,990271,996219,997855,995306,994919,999417,999610,994279,994792,991572,999651,992174,995095,995096,995210,990438,995551,991593,991172,999192,999216,999791,997340,995868\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 448\n", + "Returned IDs: 999658,994611,991088,998966,997320,990444,993253,995942,993484,991593,994249,990890,991895,996838,992546,999167,998246,990936,990949,993694,994964,996219,990210,997699,994573,998287,991155,997372,990545,997470\n", + "Ground Truth: 999658,994611,991088,998966,997320,990444,993253,995942,993484,991593,994249,990890,991895,996838,992546,999167,998246,990936,990949,993694,994964,996219,990210,997699,999247,994573,998287,991155,999991,997372\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 449\n", + "Returned IDs: 999343,990366,994964,995438,990934,995200,998921,997920,996219,997371,997842,992047,995401,990448,997192,999144,990866,998401,999647,993412,997370,996727,993289,994670,995096,997320,993237,993253,998779,995927\n", + "Ground Truth: 999343,990366,994964,995438,990934,995200,998921,997920,996219,997371,997842,992047,995401,990448,990652,997192,999144,990866,998401,999647,993412,997370,996727,993289,994670,995096,997320,997403,993237,993253\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 450\n", + "Returned IDs: 995417,997843,998653,995981,998828,997584,991467,992671,997976,992931,990066,996755,993696,992733,991131,993047,994894,994100,990780,992972,992119,992692,992635,995336,997256,995596,996075,996710,999883,992748\n", + "Ground Truth: 995417,997843,998653,995981,998828,997584,991467,992671,997976,992931,990066,996755,993696,992733,991131,993047,994894,994100,990780,992972,992119,997449,992692,991372,992635,995336,997256,995596,996075,996966\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 451\n", + "Returned IDs: 995855,999680,991392,997320,996247,999453,993300,999806,999639,996744,998232,990772,991984,993904,997777,996463,996001,995250,997258,995515,992238,993484,997380,997913,993848,990957,990352,992429,994081,990949\n", + "Ground Truth: 995855,999680,991392,997320,996247,999453,993300,999806,999639,996744,998232,990772,991984,993904,997777,996463,996001,995250,997258,995515,992238,993484,997380,997913,993848,990957,990352,992429,994081,993003\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 452\n", + "Returned IDs: 991939,993419,997785,995306,995659,998868,998921,997984,996516,990089,990366,997301,997286,991479,991999,999824,998445,990512,991852,991743,997292,998726,993143,997371,997873,994942,991406,992451,996135,990560\n", + "Ground Truth: 991939,993419,997785,995306,995659,998868,998921,997984,999516,996516,990089,990366,997301,997286,991479,991999,999824,998445,990512,991852,991743,997292,998726,993143,997371,997873,994942,991406,992451,996135\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 453\n", + "Returned IDs: 997411,996368,997868,996181,996145,993751,995702,991002,995937,998265,994730,999486,998370,997723,998990,999586,991585,990974,990412,996379,996959,999977,999537,994826,991157,995633,990132,994895,995679,995095\n", + "Ground Truth: 997411,996368,997868,996181,996145,993751,995702,991002,995937,998265,994730,999486,998370,997723,998990,999586,991585,990974,990412,996379,996959,999801,999977,999537,994826,991157,995633,999639,990132,994895\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 455\n", + "Returned IDs: 991925,996328,996506,991807,990776,997309,996318,998021,995268,990987,992726,996495,993121,994119,995080,992052,997208,995375,994895,999259,995098,991712,993180,997184,996001,997675,998566,995438,997732,992558\n", + "Ground Truth: 991925,996328,996506,991807,990776,997309,996318,998021,995268,990987,992726,991743,996495,993121,994119,995080,992052,997380,997208,995375,994895,994409,999259,995098,991712,993180,990887,997184,996001,997675\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 456\n", + "Returned IDs: 995080,990043,999784,999516,992318,995442,997918,993052,996489,996193,995119,997155,997320,995375,998726,995336,992031,997301,999453,995641,994972,991816,994162,991348,991440,990882,996840,991609,995515,993260\n", + "Ground Truth: 995080,990043,999784,999516,992318,995442,997918,993052,996489,996193,995119,996829,997155,997320,995375,998726,995336,992031,995246,999741,997301,996887,999453,995641,994972,991816,994162,993528,991348,991440\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 457\n", + "Returned IDs: 997024,995372,997470,997320,992789,994618,995841,998951,996143,998445,999109,990936,992451,992978,991362,991498,995422,990191,993241,991650,991999,992228,993788,990275,993853,993579,998764,998195,997984,998921\n", + "Ground Truth: 997024,995372,997470,997320,992789,994618,995841,998951,996143,998445,999109,990936,992451,992978,991362,991498,995422,990191,993241,991650,991999,992228,993788,990275,993853,993579,998764,998195,999929,997984\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 458\n", + "Returned IDs: 990854,991935,992450,995571,998671,996328,991731,994142,998251,990931,992713,995083,991769,998055,991521,999126,995861,999441,999775,996431,996495,992667,990593,997488,990887,991257,990712,990956,990889,996728\n", + "Ground Truth: 990854,991935,992450,995571,998671,996328,991731,994142,998251,990931,992713,995083,991769,998055,991521,999126,995861,999441,999775,996431,996495,992667,990593,997488,990887,991257,990712,990956,990889,991062\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 459\n", + "Returned IDs: 992604,997110,991052,994753,995373,998318,996488,991844,995807,991309,994643,998383,998707,991110,992387,991987,999834,997675,999136,998858,995105,999478,994677,995449,992558,991571,990915,990729,990657,993433\n", + "Ground Truth: 992604,997110,991052,994753,995373,998318,996488,991844,995807,991309,994643,998383,998707,991110,992387,991987,999834,997675,999136,998858,995105,999478,995963,994677,995449,992558,991571,990915,990729,990657\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 460\n", + "Returned IDs: 995942,999358,997468,999263,999075,990595,995771,993364,994964,996218,990560,998211,993492,990146,994741,995865,999291,998964,990958,999374,992968,999701,998644,991593,994281,993484,996592,992401,994100,993250\n", + "Ground Truth: 995942,999358,997468,999263,999075,990595,995771,993364,994964,996218,990560,998211,993492,990146,994741,995865,995502,999291,998964,996869,990958,999374,992968,999701,998644,991593,990886,994281,993484,996592\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 461\n", + "Returned IDs: 997023,991958,997544,995211,995281,990887,992917,996987,993209,992521,993561,996719,995648,995198,998720,994690,990331,996001,998990,995401,995815,998776,994742,996777,995515,993289,996371,999587,994280,996220\n", + "Ground Truth: 997023,991958,997544,995211,995281,990887,992917,996987,993209,992521,993561,996719,997163,995648,995198,998720,994690,995283,990331,996001,998990,995401,995815,998776,995911,992862,994742,997191,996777,991743\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 462\n", + "Returned IDs: 993269,992521,999990,991969,999516,995813,999866,995283,993684,992426,993561,997554,999513,994949,991869,998361,990035,993738,997822,995177,993813,995216,990603,992448,991795,994386,998910,990845,996368,998205\n", + "Ground Truth: 993269,992521,999990,991969,999516,995813,999866,995283,993684,992426,993561,997554,999513,994949,991869,998361,990035,993738,997216,997822,995177,993813,995216,990603,992448,991795,994386,998346,998910,990845\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 467\n", + "Returned IDs: 996406,992752,993347,995920,997506,995103,995961,993055,995600,996216,999205,999640,993993,992628,995051,993560,997151,996549,996483,999555,992867,998594,992293,993545,993610,998589,998716,999139,996082,998752\n", + "Ground Truth: 996406,992752,993347,995920,997506,995103,995961,993055,995600,996216,999205,999640,993993,992628,995051,993560,997151,996549,996483,999555,992867,998594,992293,993545,995185,993610,998589,998716,999139,996082\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 469\n", + "Returned IDs: 998697,995894,997586,991052,990096,994972,991713,997320,995536,997002,995740,999840,993280,998303,994040,991530,996062,992521,991332,997790,996918,997918,997246,995080,992571,994334,996368,995672,995560,993504\n", + "Ground Truth: 998697,995894,997586,991052,990096,994972,991713,997320,995536,997002,995740,999840,993280,998303,994040,991530,996062,992521,991332,997790,996918,997918,997246,995080,992571,994334,996368,995672,995560,999234\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 470\n", + "Returned IDs: 999590,996001,997920,995167,992665,992862,993157,998934,996147,994531,993174,997586,990573,998593,998299,990399,997790,996075,998481,998228,996661,999216,993168,994964,995701,991212,996028,991609,997002,995536\n", + "Ground Truth: 999590,996001,997920,995167,992665,992862,993157,998934,996147,994531,993174,997586,990573,998593,998299,997555,990399,997790,996075,998481,998228,996661,999216,993168,994964,995701,991212,996028,991609,997002\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 473\n", + "Returned IDs: 999691,991081,998864,998845,997019,990072,993366,996446,999349,993715,996106,998934,999273,995636,992752,994587,995258,993055,997039,998314,995349,991629,999498,992664,998959,999663,995270,994096,995736,995299\n", + "Ground Truth: 999691,991081,998864,998845,997019,990072,993366,996446,999349,993715,996106,998934,999273,995636,992752,994587,995258,993055,997039,998314,995349,991629,999498,992664,998959,999663,992975,995270,994096,991170\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 475\n", + "Returned IDs: 999384,990096,998495,996809,997918,993696,997906,998950,997320,998383,990210,991571,996805,992944,993018,995438,993260,993215,993154,996623,999210,993744,994661,999390,991801,993504,999009,991743,990843,999320\n", + "Ground Truth: 999384,990096,998495,996809,997918,993696,997906,998950,997320,998383,990210,991571,996805,992944,993018,998861,995438,993260,993154,993215,996623,999210,993744,994661,999390,991801,993504,999009,991743,990843\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 477\n", + "Returned IDs: 990866,997364,993168,997586,994782,996232,993642,993831,994964,997732,998107,997493,997279,998668,993878,995906,993689,995482,991325,998369,997268,995107,993630,998837,992862,997305,997091,991595,995944,997004\n", + "Ground Truth: 990866,997364,993168,997586,996352,994782,996232,993642,993831,994964,997732,998107,997493,997279,998668,993878,990690,995906,993689,995482,991325,998369,996647,997268,995107,993630,998837,992862,990934,997305\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 480\n", + "Returned IDs: 994964,997913,992546,994100,995981,995942,997320,997470,999444,994778,990231,992330,999109,995560,990588,996219,991721,999374,999740,997301,993168,998540,993484,999328,998495,998797,999900,990925,996075,996809\n", + "Ground Truth: 994964,997913,992546,994100,995981,995942,997320,997470,999444,994778,990231,992330,999109,995560,990588,996219,991721,999374,999740,997301,993168,998540,993484,999328,998495,998797,999900,990925,998163,996075\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 481\n", + "Returned IDs: 999860,992558,996609,996196,996523,993839,997002,997436,991334,992411,992161,993583,996606,995618,994418,996389,997003,991977,990701,997864,999240,999715,996429,996127,999540,996882,997457,997914,999374,990249\n", + "Ground Truth: 999860,992558,996609,996196,996523,993839,997002,997436,991334,992411,992161,993583,996606,995618,994418,996389,997003,995482,991977,990701,998679,997864,999240,999715,996429,996127,999540,996882,997457,997914\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 485\n", + "Returned IDs: 991610,992778,997009,990293,999555,999994,992136,992721,990167,995257,993076,991667,990177,992862,994374,998383,996408,991513,991987,998586,999430,994796,997506,990437,997752,991304,991569,996633,997655,995536\n", + "Ground Truth: 991610,992778,997009,990293,999555,999994,992136,992721,990167,995257,993076,991667,990177,992862,994374,998383,996408,991513,991987,998586,999430,994796,997506,990437,997752,991304,991569,993993,999216,996633\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 488\n", + "Returned IDs: 997270,990039,996475,998794,996650,997047,992780,990377,992100,993479,995750,999430,999323,996013,996830,999659,993890,999551,990718,991959,993391,992872,998599,992754,993721,991382,999722,994558,997331,996016\n", + "Ground Truth: 997270,990039,996475,998794,996650,997047,992780,990377,992100,993479,995750,999430,999323,996013,996830,999659,997939,993890,999551,990718,991959,993391,992872,998599,992754,993721,991382,999722,994558,997331\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 491\n", + "Returned IDs: 990096,998968,998419,993624,994636,992716,996386,991936,999103,991167,997884,999711,999119,996209,999282,998861,990072,990154,999287,995536,997803,991451,999319,993055,993744,999443,991724,996283,999384,996893\n", + "Ground Truth: 990096,998968,998419,993624,994636,992716,996386,991936,999103,991167,997884,999711,999119,996209,999282,998861,990072,990154,999287,995536,997803,991451,999319,993055,999443,993744,991724,996283,996429,999384\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 494\n", + "Returned IDs: 992169,996325,997852,992152,997476,992632,992813,992309,994922,993717,996093,990490,999676,990266,994318,999843,999453,994761,996438,992612,990399,997822,994148,999077,995210,997797,998171,994956,990268,991334\n", + "Ground Truth: 992169,996325,997852,992152,997476,992632,992813,992309,994922,993717,996093,990490,999676,990266,994318,999843,999453,994761,996438,992612,997645,990399,997822,994148,994161,999077,995210,997797,998171,994956\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 496\n", + "Returned IDs: 992124,991984,995516,998697,994742,999136,995701,998758,993186,993727,995672,995562,993470,992747,993834,996075,994374,993342,996657,999041,991530,994082,991052,994809,999783,994879,998586,998950,995203,995981\n", + "Ground Truth: 992124,991984,995516,998697,994742,999136,995701,998758,993186,993727,995672,995562,993470,992747,993834,996075,994374,993342,996657,999041,991530,994082,991052,994809,999783,997377,994879,998586,998950,995203\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 498\n", + "Returned IDs: 999899,994906,996759,992648,992651,999977,998083,990304,991175,998301,992979,993713,992126,993275,996368,997059,999820,996714,996425,993032,996075,997554,993280,994860,997910,999587,999453,994334,994946,997002\n", + "Ground Truth: 999899,994906,996759,992648,992651,999977,998083,990304,990260,991175,998301,992979,993713,992126,993275,996368,997059,999820,996714,996425,993032,996075,997554,993280,994860,997910,999587,999453,994334,994946\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 499\n", + "Returned IDs: 998080,991334,999479,998820,991025,995190,993473,993931,997395,994222,990126,998493,994204,995612,991449,991743,990733,996609,993929,997212,992685,995401,999098,990249,992604,997515,995332,995289,997723,999788\n", + "Ground Truth: 998080,991334,999479,998820,991025,995190,993473,993931,997395,994222,990126,997030,998493,999586,994204,995612,991449,991743,990733,996609,993929,997212,992685,995401,999098,990249,992604,997515,995332,995289\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 500\n", + "Returned IDs: 999009,991776,995895,990096,993397,997586,990930,990786,999430,995455,994742,996624,990866,990701,990271,997028,996830,997768,999599,991154,995701,996216,998495,996192,995435,999639,991572,996001,999647,997378\n", + "Ground Truth: 999009,991776,995895,990096,993397,997586,990930,990786,999430,995455,994742,996624,990866,990701,990271,997028,996830,997768,999599,991154,995701,996216,998495,996192,995435,999639,995899,991572,996001,999647\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 501\n", + "Returned IDs: 998158,997453,994463,992488,996062,998874,992322,990930,996915,991021,994783,995289,999139,991154,993561,993673,996071,990076,990104,991685,990226,995113,999091,995783,997797,996028,991857,992703,995586,992182\n", + "Ground Truth: 998158,997453,994463,992488,996062,998874,992322,990930,996915,991021,994783,995289,999139,991154,994777,993561,993673,996071,990076,990104,991685,990226,995113,999091,992326,995783,997797,996028,991857,992703\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 503\n", + "Returned IDs: 994894,998353,999599,995981,994100,997913,995942,992228,997946,999243,999658,991045,990701,997301,994266,998090,997728,998246,999374,996219,996805,993186,992384,997320,999331,997614,997304,999444,992848,993529\n", + "Ground Truth: 994894,998353,999599,995981,994100,997913,995942,992228,997946,999243,999658,991045,990701,997301,994266,998090,997728,998246,999374,996219,991468,996805,993186,992384,997320,999331,997614,997304,999444,992848\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 504\n", + "Returned IDs: 990532,999453,995852,994081,995130,994955,993053,995295,996147,992408,994042,995203,995456,990392,990771,991609,999052,998039,997002,999639,996918,997918,993809,998855,993906,994226,996368,992832,992562,996001\n", + "Ground Truth: 990532,999453,995852,994081,995130,994955,993053,995295,996147,992408,994042,995203,995456,990771,990392,991609,999052,998039,997002,999639,996918,995401,997918,993809,998855,993906,994226,996368,992832,992562\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 505\n", + "Returned IDs: 991698,997395,992790,990215,994671,991404,993409,992832,995375,995080,991607,994855,997515,990889,995685,992576,991576,995648,996245,992749,999052,990930,997391,998536,993412,995968,998158,997192,992036,995994\n", + "Ground Truth: 991698,997395,993407,992790,990215,994671,991404,993409,992832,995375,995080,991607,994855,997515,990889,995685,992576,991576,995648,996245,992749,999052,990930,997391,998536,993412,995968,998158,997192,992036\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 507\n", + "Returned IDs: 994472,999791,991921,990089,992322,994402,990091,995222,992820,995720,998107,997939,999876,998765,996355,990622,998355,990226,999642,998630,995535,998594,996761,993775,993584,997349,995638,993929,991325,991751\n", + "Ground Truth: 994472,999791,991921,990089,992322,994402,990091,995222,992820,995720,998107,997939,999876,998765,996355,990622,996050,998355,990226,999642,998630,995535,998594,996761,993775,993584,997349,995638,993929,991325\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 508\n", + "Returned IDs: 992582,991743,990949,991782,990958,995274,995374,999533,993961,997614,994850,990490,994266,995949,995981,997645,991479,996648,999331,990957,997913,990588,990359,990493,991889,993431,999343,990910,998294,993412\n", + "Ground Truth: 992582,991743,990949,991782,990958,995274,995374,999533,993961,997614,994850,990490,994266,995949,995981,997645,991479,996648,999331,990957,997913,990588,990359,990493,991889,999328,993431,999343,990910,998294\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 509\n", + "Returned IDs: 998982,990328,992226,990528,990701,992068,998671,996939,992230,991334,990468,990711,998719,994751,999500,994783,994689,999916,995295,991771,997379,997586,990126,996116,990011,990864,991075,991464,996645,994558\n", + "Ground Truth: 998982,990328,992226,990528,990701,996110,992068,998671,996939,992230,991334,990468,990711,998719,994751,999500,994783,994689,991239,999916,995990,995295,991771,997379,997586,990126,996116,990011,990864,991075\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 510\n", + "Returned IDs: 997913,990650,991593,998500,994964,997301,994831,997320,998170,999982,997192,992978,994618,990066,995942,999948,999167,992087,998899,995802,996219,991335,997470,999647,996504,998550,995961,996053,990210,995949\n", + "Ground Truth: 997913,990650,991593,998500,994964,997301,994831,997320,998170,999982,997192,992978,994618,990066,995942,999948,999167,992087,998899,995802,996219,991335,997470,998808,999647,996504,998550,995961,996053,990210\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 511\n", + "Returned IDs: 993387,994192,998899,992464,992119,997913,996592,990957,991479,990890,997320,993662,995336,994978,999658,996296,996075,993853,992330,999559,998550,998090,991131,994432,999948,990545,995826,992969,992255,991379\n", + "Ground Truth: 993387,994192,998899,996946,992464,992119,997913,996592,990957,991479,990890,997320,993662,992134,993174,995336,994978,999658,996296,996075,993853,992330,991260,999559,998550,998090,991131,994432,999948,996397\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 512\n", + "Returned IDs: 997168,996277,999281,991245,993275,999526,992315,994743,993715,990409,991685,992636,996001,998623,999724,990956,991349,991037,998536,993560,994016,990951,998045,995701,993561,995562,995835,993738,996677,995408\n", + "Ground Truth: 997168,996277,999281,991245,993275,999526,992315,994743,993715,990409,991685,992636,996001,998623,999724,990956,991349,991037,998536,993560,994016,990951,998045,995701,993561,995562,991755,995835,993738,994409\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 513\n", + "Returned IDs: 991375,993906,996216,996082,997790,995991,994925,996062,998950,996599,996178,995107,991769,995895,996001,995961,999827,990786,990364,999009,993561,990886,992780,995705,994280,993931,991142,995586,992847,990096\n", + "Ground Truth: 991375,993906,996216,996082,997790,995991,994925,996062,998950,996599,996178,995107,991769,995895,992473,996001,995961,999827,990786,990364,999009,993561,990886,992780,995705,994280,993931,990887,991142,995586\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 514\n", + "Returned IDs: 991396,994160,990925,998797,990595,997972,999262,996959,995884,994964,997102,993914,995588,996532,990146,998501,993168,997040,999600,994673,993389,997320,993761,997189,990863,991008,993083,991086,990797,992284\n", + "Ground Truth: 991396,994160,990925,998797,990595,997972,999262,996959,995884,994964,997102,993914,995588,996532,990146,998501,993168,997040,997028,999600,994673,993389,997320,995909,993761,997189,990863,991008,993083,991086\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 515\n", + "Returned IDs: 996379,991255,995268,990930,995963,993561,992367,991747,991603,996913,991459,997925,990984,993673,992703,996198,996277,992294,994589,992115,995701,990133,996867,990899,994113,995289,991011,998779,996328,996918\n", + "Ground Truth: 996379,991255,995268,990930,995963,993561,992367,991747,991603,996913,991459,997925,990984,993673,992703,996198,996277,992294,992660,994589,990897,992115,995701,990133,996867,990899,994113,995289,991011,998779\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 517\n", + "Returned IDs: 996038,998832,995507,997723,995139,997046,996323,995905,999619,997406,996867,992545,993433,999000,998165,993421,999530,995289,992130,997832,996769,997020,996112,991027,993390,996166,996815,995753,992605,995052\n", + "Ground Truth: 996038,998832,995507,997723,995139,997046,996323,995905,999619,997406,996867,992545,993433,999000,998165,993421,999530,995289,992130,997832,996769,997020,996112,991027,993390,996166,992660,996815,995753,992605\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 520\n", + "Returned IDs: 992692,997320,997918,993512,990171,992521,998260,995336,999731,993071,994964,993492,994360,999516,996397,994081,998365,998246,994290,995826,992124,996417,995274,997977,992330,992909,991883,991880,992047,998568\n", + "Ground Truth: 992692,997320,997918,993512,990171,992521,998260,995336,999731,993071,994964,993492,994360,999516,996397,994661,994081,998365,998246,994290,995826,992124,996417,995274,997977,992330,992909,991883,991880,992047\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 521\n", + "Returned IDs: 999453,994081,990661,990532,996075,997370,990652,998567,998492,998734,996309,990882,995515,997320,996626,992521,995372,991713,993280,995203,991869,994789,990083,999954,996368,995401,992031,995245,998607,996966\n", + "Ground Truth: 999453,994081,990661,990532,996075,997370,990652,998567,998492,998734,996309,990882,995515,997320,996626,992521,996193,995372,991713,993280,995203,991869,999123,994789,990083,999954,996368,995401,992031,995245\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 523\n", + "Returned IDs: 998246,992896,990066,997320,993484,994225,996560,998540,999102,998634,999128,995904,995224,999005,992986,995400,994542,992839,998384,997610,995841,998603,990269,996598,997178,994164,994214,991999,994964,992054\n", + "Ground Truth: 998246,992896,990066,997320,993484,994225,996560,999757,998540,999102,998634,999128,995904,995224,999005,992986,995400,994542,992839,991567,997102,998384,997610,995841,998603,990269,996598,997178,994164,994214\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 524\n", + "Returned IDs: 990882,994612,997465,999698,998937,997403,993845,998607,995515,999036,996309,992228,994922,995788,997164,995154,993002,992309,996840,993028,998921,999671,998244,992843,993143,991510,995893,996653,998401,997486\n", + "Ground Truth: 990882,994612,997465,999698,998937,997403,993845,998607,995515,999036,996309,992228,994922,995788,997164,995154,993002,992309,996840,993028,998921,999671,998244,992843,993143,997591,991510,995893,996653,998401\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 527\n", + "Returned IDs: 995596,993341,996741,990697,997305,996075,994886,992041,994985,993785,996178,990571,993119,994859,996755,997661,994440,994971,993904,992661,990022,993696,996210,997291,992635,993741,991218,991650,997320,998715\n", + "Ground Truth: 995596,993341,996741,990697,997305,996075,994886,992041,994985,993785,996178,990571,993119,994859,996755,993154,997661,994440,994971,993904,992661,990022,997699,993696,996210,997291,992635,993741,991218,991650\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 529\n", + "Returned IDs: 992636,997168,995837,997201,995295,993715,994621,999052,998076,995159,994488,997838,991451,990956,992832,992595,996505,994380,990336,990887,990889,996147,995836,993448,996277,992973,998174,995521,991007,997329\n", + "Ground Truth: 992636,997168,995837,997201,995295,993715,994621,999052,998076,995159,994488,997838,991451,990956,992832,992595,996505,994380,998021,990336,990887,990889,996147,995836,993448,996277,992973,998174,995521,991007\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 533\n", + "Returned IDs: 997506,992154,996775,996055,995226,994891,997944,997050,991633,998933,998177,999139,999079,993665,999576,993347,991712,998536,992793,990194,996817,992170,997666,997906,994955,994280,996733,995536,999091,993074\n", + "Ground Truth: 997506,992154,996775,996055,995226,994891,997944,997050,991633,998933,998177,999139,999079,993665,999576,993347,991712,998536,992793,990194,996817,990959,992170,997666,997906,994955,994280,996733,995536,999091\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 534\n", + "Returned IDs: 999876,993484,993504,998302,990092,993803,997320,998294,996869,997370,997913,998029,999363,991498,993741,994826,998580,992384,993885,999535,999374,990146,991043,991593,998246,994089,994366,995949,995372,997661\n", + "Ground Truth: 999876,993484,993504,998302,990092,993803,997320,998294,996869,997370,997913,998029,999363,991498,993741,994826,998580,992384,993885,999535,999374,990146,991043,991593,999982,998246,994089,994366,995949,995372\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 541\n", + "Returned IDs: 998965,996219,994452,992998,995339,990780,999683,995455,996111,994777,991370,996178,992070,997032,997004,995460,990886,996612,998364,990931,990366,993128,997813,999343,998390,999013,997706,998593,999876,991179\n", + "Ground Truth: 998965,996219,994452,992998,995339,990780,999683,995455,996111,994777,991370,996178,992070,997032,997004,995460,994981,990886,996612,998364,991373,990931,990366,993128,999343,997813,993690,998390,999013,997706\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 543\n", + "Returned IDs: 992944,993018,995560,998950,991142,995438,996726,998160,996140,990375,990843,991650,993324,998364,997320,991951,995107,999210,994171,994964,995130,999647,997379,995332,996370,998697,998180,993499,996653,996810\n", + "Ground Truth: 992944,993018,995560,998950,991142,995438,996726,998160,996140,990375,990843,991650,993324,998364,997320,991951,995107,999210,994171,994964,995130,999647,997379,995332,996370,999109,998697,998180,993499,996653\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 544\n", + "Returned IDs: 993755,992761,991850,998778,991779,995222,997155,992595,995536,994356,992832,999107,999256,993055,995385,996001,996216,998536,992110,997158,990127,995501,991474,991644,995655,996328,999798,992561,997586,997141\n", + "Ground Truth: 993755,992761,991850,998778,991779,995222,997155,992595,995536,994356,992832,999107,999256,993055,995385,996001,996216,998536,992110,997158,990127,995501,991474,999384,991644,995655,996328,992224,999798,992561\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 545\n", + "Returned IDs: 997391,992428,991698,996930,992933,990438,998080,998855,994193,991619,993255,991257,993288,997918,991337,992179,995357,991743,992850,996762,995511,995401,994062,992685,999144,998180,992516,990459,996903,992531\n", + "Ground Truth: 997391,992428,991698,996930,992933,990438,998080,998855,994193,991619,993255,991257,993288,997918,991337,992179,999527,995357,991743,992850,996762,995511,995401,994062,999144,992685,998180,992516,990459,996903\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 546\n", + "Returned IDs: 994419,996745,992894,990252,997776,996322,996148,998593,992660,995643,997046,998486,997940,995289,992125,995470,991301,999482,995923,996867,992048,997822,998045,993412,995702,991958,995063,998564,995401,996323\n", + "Ground Truth: 994419,996745,992894,990252,997776,996322,996148,998593,992660,995643,997046,998486,997940,995289,992125,995470,991301,999482,995923,996867,992048,997822,998045,993412,995702,991958,995063,998564,998105,995401\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 548\n", + "Returned IDs: 990092,995716,990315,999261,994922,999620,990701,990934,997371,990648,996916,991921,997770,991196,999613,991588,994990,995222,995401,995693,998641,992944,992384,997403,997855,996219,991999,995551,999343,996840\n", + "Ground Truth: 990092,995716,998991,990315,999261,994922,999620,997920,990701,990934,997371,990648,996916,991921,997770,991196,999613,991588,994990,995222,995401,995693,998641,992944,992384,996419,997403,997855,996219,999151\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 549\n", + "Returned IDs: 997897,998250,993483,999463,992437,995421,992727,998593,990255,990521,992367,998934,993175,995626,998628,992407,996181,991245,998162,993489,994280,995401,990844,998032,996001,994809,997918,993727,996657,993232\n", + "Ground Truth: 997897,998250,993483,999463,992437,995421,992727,998593,990255,990521,992367,998934,993175,995626,998628,992407,996181,991245,998162,993489,994280,995182,995401,990844,998032,996001,994809,997918,993727,996657\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 551\n", + "Returned IDs: 991335,998500,990650,997006,995985,997688,999647,995942,994776,998139,998329,990092,994699,997192,992357,996727,997371,995370,992546,990934,990366,999898,995906,990980,993034,994778,990866,992994,994889,990315\n", + "Ground Truth: 991335,998500,990650,997006,995985,997688,993174,999647,995942,994776,998139,998329,990092,994699,997192,992357,996727,997371,995370,992546,990934,990366,999898,995906,990980,992822,993034,994778,990866,996576\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 552\n", + "Returned IDs: 995655,991667,997973,992517,997145,993001,991685,999107,990330,990458,991093,995222,993280,998861,990091,998122,997732,997679,992561,998536,997265,991569,996733,996028,992862,995835,997332,999827,995780,995536\n", + "Ground Truth: 995655,991667,997973,992517,997145,993001,991685,999107,990330,990458,991093,995222,993280,998861,990091,998122,997732,997679,992561,998536,997265,991569,996733,996028,998710,992862,995835,997332,999827,993006\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 557\n", + "Returned IDs: 995336,999900,992277,998090,992384,996296,990949,994100,996657,996001,997918,997320,998951,995873,997571,993143,995374,993809,997554,997913,990772,992330,993512,990339,996592,992880,992052,993492,999144,997156\n", + "Ground Truth: 995336,999900,992277,998090,992384,996296,990949,994100,996657,996001,997918,997320,998951,990343,995873,997571,993143,995374,993809,997554,997913,990772,992330,993512,990339,996592,992880,992052,993492,999144\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 558\n", + "Returned IDs: 998934,998720,991657,991459,994167,994280,991130,996001,998593,992429,999639,999173,999216,993834,999408,995103,996408,992437,992441,993993,995158,995701,993375,996328,997790,992225,996526,993848,998864,997847\n", + "Ground Truth: 998934,998720,991657,991459,994167,994280,991130,996001,992429,998593,999639,999173,999216,993834,999408,995103,996408,992437,992441,993993,997337,995158,995701,993375,996328,997790,992225,996526,997166,993848\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 561\n", + "Returned IDs: 991753,995066,995401,996607,993003,991680,994106,996813,992448,997975,996304,990319,999054,991639,995864,998910,997544,993642,997091,996695,999759,997176,999646,997075,991795,995702,995490,992653,993209,991592\n", + "Ground Truth: 991753,995066,995401,996607,993003,991680,994106,996813,992448,997975,996304,990319,999054,993597,991639,995864,998910,997544,993642,997091,996695,999759,999646,997176,997075,991795,995702,995490,992653,993209\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 563\n", + "Returned IDs: 990508,998359,999717,990031,995206,991323,991650,993603,997581,991348,996178,992001,999132,996075,999258,993686,996626,994233,993809,999806,999473,994341,996458,990022,992557,990866,997442,990912,997367,999480\n", + "Ground Truth: 990508,998359,999717,990031,995206,991323,991650,993603,997581,991348,996178,992001,999132,996075,999258,993686,996803,996626,994233,993809,999806,999473,994341,996458,990022,992557,990866,997442,990912,997367\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 564\n", + "Returned IDs: 993591,997036,993015,993553,994275,997170,991989,996906,990368,994021,995923,995345,992585,998039,994663,990955,992798,994386,993561,993164,994379,992014,990845,993725,997357,993813,991168,993697,997933,995076\n", + "Ground Truth: 993591,997036,993015,993553,994275,997170,991989,996906,990368,994021,995923,995345,992585,998039,994663,990955,992798,994386,993561,993164,994379,992014,990845,993725,997357,993813,997920,991168,993697,997933\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 565\n", + "Returned IDs: 992286,993030,995854,997634,997020,994301,998386,995723,997114,996273,998080,993263,992683,998650,990591,992531,994677,997723,991423,996139,997395,990103,993717,990189,991404,995062,993223,995625,993127,990178\n", + "Ground Truth: 992286,993030,995854,997634,997020,994301,998386,995723,997114,996273,998080,993263,992683,998650,995282,990591,992531,994677,997723,994360,991423,996139,997395,990103,993717,990189,991404,995062,993223,995625\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 567\n", + "Returned IDs: 990974,994541,994557,993637,999009,996996,992551,999898,994638,992553,993807,997320,990096,997004,996216,993695,995906,993928,998633,991179,992927,998006,995895,990786,997410,996564,996944,995222,997732,998302\n", + "Ground Truth: 990974,994541,994557,993637,999009,996996,992551,999898,994638,992553,993807,997320,990096,997004,996216,993695,995906,993928,998633,996830,991179,992927,998006,997577,995895,990786,997410,995755,996564,996944\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 569\n", + "Returned IDs: 994145,993561,993993,993939,996328,992862,996001,990887,994853,998710,995326,995535,992450,997878,996219,996132,997529,994742,999472,990096,994682,996408,991759,999444,998382,993342,996042,995895,994830,990918\n", + "Ground Truth: 994145,993561,993993,993939,996328,992862,996001,990887,994853,998710,995326,995535,992450,997878,996219,996132,997529,994742,999472,990133,990096,994682,996408,991759,999444,998382,993342,996042,995895,994830\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 571\n", + "Returned IDs: 993997,998962,990663,999279,997031,996786,997844,990758,996131,995633,996867,998609,996302,993920,993767,995640,995268,990157,994120,999996,993209,993488,997566,995766,995289,996346,999415,996109,991214,997223\n", + "Ground Truth: 993997,998962,990663,999279,997031,996786,997844,990758,996131,995633,996867,998609,996302,993920,993767,995640,995268,990157,994120,999996,993412,993209,993488,997566,995766,995289,996346,999415,996254,996109\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 574\n", + "Returned IDs: 991348,995206,995108,996626,992318,993603,993053,995438,990083,998950,996024,991763,993621,994233,992944,991142,993166,996178,994742,993154,992617,998567,990031,995107,998180,996001,993696,997448,999426,995669\n", + "Ground Truth: 991348,995206,995108,996626,992318,993603,993053,995438,990083,998950,996024,991763,993621,994233,992944,991142,993166,993747,996178,994742,993154,992617,998567,990031,995107,998180,996001,993696,997448,999426\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 575\n", + "Returned IDs: 990450,998383,999136,998570,998586,994105,991844,999306,996446,995695,995257,996001,995142,994643,998593,996043,993948,990991,994185,991987,999653,994187,995455,997191,999117,995701,999872,991038,992945,995755\n", + "Ground Truth: 990450,998383,999136,998570,998586,994105,991844,999306,996446,995695,995257,996001,995142,994643,998593,996043,993948,990991,994185,991987,999653,994187,997766,995455,997191,998493,999117,995701,999872,991038\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 576\n", + "Returned IDs: 998623,990998,998536,995408,992052,994743,990096,999287,990255,996809,995203,997618,995536,994682,992558,991630,999107,994742,995401,994510,991629,991515,992933,994955,993910,999724,996483,995198,994954,991977\n", + "Ground Truth: 998623,990998,998536,995408,992052,994743,990096,999287,990255,996809,995203,997618,995536,994682,992558,991630,999107,995864,994742,995401,994510,991629,991515,992933,994955,993910,999724,996483,995198,994954\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 577\n", + "Returned IDs: 997918,993223,996903,995373,993512,995453,994088,991887,995444,996282,991844,994334,992604,994092,991698,995807,993412,994804,997675,998318,995685,991110,997515,997637,992933,999144,997110,999478,994819,996867\n", + "Ground Truth: 997918,993223,996903,995373,993512,995453,994088,991887,995444,996282,991844,995080,994334,992604,994092,991698,995807,993412,994804,997675,998318,995685,991110,997515,997637,992933,999144,997110,999478,994819\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 578\n", + "Returned IDs: 997110,991987,995268,999478,994948,990991,997723,994643,993122,995848,998078,990805,991630,999994,990361,993381,990930,994125,997191,996968,991052,999117,996867,991954,999530,995963,994253,995701,999555,998257\n", + "Ground Truth: 997110,991987,995268,999478,994948,990991,997723,994643,993122,995848,998078,990805,991630,999994,990361,993381,990930,994125,997191,996968,991052,999117,996867,991954,999530,995963,994253,995701,992556,999555\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 581\n", + "Returned IDs: 995899,991154,998697,992167,998720,995536,993139,994167,995562,996075,999806,999724,996811,997305,999639,996001,992225,994577,997586,992423,993560,991157,995963,996894,999107,993055,995203,998514,998568,994636\n", + "Ground Truth: 995899,991154,998697,992167,998720,995536,993139,994167,995562,996075,999806,999724,996811,997305,999639,996001,992225,994577,997586,992423,993560,991157,995963,996894,999107,993055,998105,995203,998514,998568\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 583\n", + "Returned IDs: 992680,995756,996645,993725,991669,999740,999356,991075,997798,992090,993444,999302,990743,999922,995435,991464,996411,993645,990528,992024,999261,990701,999280,992783,996627,993205,996969,998940,993053,996384\n", + "Ground Truth: 992680,995756,996645,993725,991669,999740,999356,991075,997798,992090,993444,999302,990743,999922,995435,991464,996411,991624,993645,990528,992024,999261,990701,999280,992783,996627,993205,996969,998940,993053\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 584\n", + "Returned IDs: 998182,994799,990328,998105,993857,999460,991075,990980,998017,995166,998427,998197,996689,995295,996028,997961,995823,990145,999843,993055,992480,999230,993475,992374,994119,998982,996494,990536,990097,990811\n", + "Ground Truth: 998182,994799,990328,998105,993857,999460,991075,990980,998017,995166,998427,998197,996689,995295,996028,997961,995823,990145,999843,993055,992480,999230,993475,992374,994119,998982,996494,990536,990854,990097\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 586\n", + "Returned IDs: 992183,991479,995802,998011,992946,994778,993853,998550,994699,995396,990231,997439,991207,990647,994450,993662,999243,993403,995465,996912,997913,999647,996735,991147,990478,999710,995793,997194,991502,998899\n", + "Ground Truth: 992183,991479,990109,995802,998011,992946,994778,999561,993853,998550,994699,995396,990231,997439,991207,990647,994450,993662,999243,993403,995465,996912,997913,999647,996735,991147,990478,999710,995793,997194\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 588\n", + "Returned IDs: 990845,994925,996906,998593,992703,997453,998039,993561,991958,995162,996062,996001,999729,995401,991789,993906,991989,992334,996237,997028,990899,991011,990096,990076,995289,996311,996277,999340,998207,994424\n", + "Ground Truth: 990845,994925,996906,998593,992703,997453,998039,993561,991958,995162,996062,996001,999729,995401,991789,993906,991989,992334,996237,997028,990899,991011,990096,990076,995289,996311,996277,999340,998207,995560\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 589\n", + "Returned IDs: 990271,997878,990179,995222,991776,995868,990621,994448,992193,996219,998269,995719,994145,990369,996580,994689,994373,990272,997349,995096,991075,998204,990077,999102,990315,995438,995690,994682,994742,993725\n", + "Ground Truth: 990271,997878,990179,995222,991776,995868,990621,994448,992193,996219,998269,995719,994145,990369,996580,994689,994373,990272,997349,996593,995096,991075,998204,990077,999102,990315,995438,995690,994682,994742\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 591\n", + "Returned IDs: 993433,996042,995289,996607,992448,998910,995401,995146,993561,995820,995018,995200,996769,995521,998162,995864,996112,996815,991671,999611,992576,991699,996867,997723,991958,994409,997822,995766,999054,997406\n", + "Ground Truth: 993433,996042,995289,996607,992448,998910,995401,995146,993561,996809,995820,995018,997023,995200,996769,995521,998162,995864,996112,996815,991671,999611,992576,991699,996867,997723,991958,994409,997822,995766\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 593\n", + "Returned IDs: 992558,993255,994280,992790,995401,990790,997395,995374,995886,995134,990214,991404,999715,995145,996762,993478,991977,997918,990538,992685,999305,995119,995154,995968,991052,990969,992933,996803,999179,999154\n", + "Ground Truth: 992558,993255,994280,992790,995401,990790,997395,995374,995886,995134,990214,991404,999715,995145,996762,993478,991977,997918,990538,992685,999305,995119,996721,995154,995968,991052,998045,990969,992933,999461\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 595\n", + "Returned IDs: 991828,992186,990580,991840,990310,997539,994922,999076,996402,996996,993036,993577,994005,990138,995515,996449,996478,991609,991172,999846,996840,993143,992207,991519,998455,995156,995767,998711,991072,994917\n", + "Ground Truth: 991828,992186,990580,991840,990310,997539,994922,999076,996402,996996,993036,993577,994005,990138,995515,996449,996478,994700,991609,991172,999846,993278,996840,993143,992207,991519,998455,995156,995767,998711\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 596\n", + "Returned IDs: 995981,997620,999374,997304,997437,998246,995792,995596,992357,994266,999331,997913,998697,992748,994100,995132,990588,997429,993186,994916,997320,994894,995638,990949,997614,996075,992384,999815,998353,991045\n", + "Ground Truth: 995981,997620,999374,997304,997437,998246,995792,995596,992357,994266,999331,997913,998697,992748,992047,999710,994100,995132,990588,997429,993412,993186,994916,992583,997320,998974,994894,995638,990949,997614\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 600\n", + "Returned IDs: 996610,997787,991603,998906,996047,996388,990033,993390,995289,991463,997406,990681,998045,990940,994161,998165,993561,996112,992125,991367,991954,996769,998999,995182,992160,997046,995497,997832,995318,995052\n", + "Ground Truth: 996610,997787,991603,998906,996047,996388,990033,993390,995289,991463,997406,990681,998045,990940,994161,998165,995040,993561,996112,992125,991367,991954,996769,998999,995182,992160,997046,995497,997832,995318\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 604\n", + "Returned IDs: 994297,995244,994964,993452,991438,994919,998500,999261,999821,998837,999444,992169,995135,996854,995591,998909,991991,999545,999740,994922,991999,990736,990432,997984,996351,991840,998553,995336,994898,997720\n", + "Ground Truth: 994297,995244,994964,993452,991438,994919,998500,999261,999821,998837,999444,990064,992169,995135,996854,995591,998909,991991,999545,999740,994922,991999,990736,990432,997984,996351,991840,998553,995336,991751\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 605\n", + "Returned IDs: 999375,997152,992468,991960,998196,994193,991758,994119,999788,998274,992056,991447,995438,991515,998543,994742,994541,995494,997452,990814,991334,999981,990127,993180,997967,992077,998913,990796,996302,999543\n", + "Ground Truth: 999375,997152,992468,991960,998196,994193,991758,994119,999788,998274,992056,991447,998466,995438,991515,998543,994742,994541,995494,997452,990814,991334,999981,990127,993180,997967,992077,998913,990796,996302\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 610\n", + "Returned IDs: 991638,997732,994956,993929,997112,993765,992655,998107,992120,995861,992813,999827,992169,998458,992408,990226,997852,990441,991334,994131,993877,990889,990961,997453,990266,996328,994557,998495,997822,993080\n", + "Ground Truth: 991638,997732,994956,993929,997112,993765,992655,998107,992120,995861,992813,999985,999827,992169,998458,992408,990226,997852,990441,991334,994131,993877,990889,990961,997453,990266,996328,994557,998495,997822\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 611\n", + "Returned IDs: 999056,999426,996744,996500,992167,996075,991984,997777,992041,992927,993766,995291,991823,993283,993156,997661,995596,996761,995132,993237,999041,999132,996169,996001,995516,995898,994809,993834,998990,999219\n", + "Ground Truth: 999056,999426,996744,996500,992167,996075,991984,997777,992041,992927,993766,995291,991823,993283,993156,997661,995596,996761,995132,993237,995375,999041,999132,996169,996001,995516,995898,994809,993834,998990\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 612\n", + "Returned IDs: 994830,994742,998950,993588,996809,992807,998913,998823,994806,995552,994269,992318,996805,991154,996301,999627,990392,991390,993342,998495,995299,999430,992598,992551,994751,992427,993053,995899,995002,994682\n", + "Ground Truth: 994830,994742,998950,993588,996809,992807,998913,998823,994806,995552,994269,992318,996805,991154,996301,999627,990392,991390,993342,998495,995299,999430,992598,992551,994751,992427,993053,995899,995401,995002\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 613\n", + "Returned IDs: 994051,990256,991860,995516,997645,992915,997811,991799,995755,990862,992596,992165,995132,997429,993396,993336,997305,996856,996811,999251,991052,994879,992357,995963,991690,995184,996406,990958,996942,991982\n", + "Ground Truth: 994051,990256,991860,995516,997645,992915,997811,991799,995755,990862,992596,992165,997437,995132,997429,993396,999725,993336,992238,997305,996856,996811,999251,994335,991052,994879,992357,995963,990178,992546\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 615\n", + "Returned IDs: 996116,990701,997374,994350,997207,998913,997209,992024,994120,990126,995704,991156,994168,995482,997628,992599,992251,992871,993240,993384,990614,994730,999402,992156,993857,994728,995633,998982,991943,996481\n", + "Ground Truth: 996116,990701,997374,994350,997207,998913,997209,992024,994120,990126,995704,999586,991156,994168,995482,997628,992599,992251,992871,993240,993384,990614,994730,999402,992156,993857,994728,995633,998982,991943\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 616\n", + "Returned IDs: 998566,997193,995691,997126,990305,994064,993566,996194,990184,996451,991636,996421,993127,998466,995482,990189,996170,990776,993678,999025,992841,995883,995620,992944,990948,993920,999158,992067,993181,994351\n", + "Ground Truth: 998566,997193,995691,997126,990305,994064,993566,996194,990184,996451,991636,996421,993127,998466,995482,990189,991157,996170,990776,993678,999025,999586,992704,991938,992841,995883,990827,990052,995620,992944\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 618\n", + "Returned IDs: 998888,999555,991944,990991,999584,991100,992638,991188,997154,997850,996393,995655,990167,998006,993948,993749,990423,990558,990154,994727,994467,993381,997030,993956,990361,994374,994643,991385,991596,998257\n", + "Ground Truth: 998888,999555,991944,990991,999584,991100,992638,991188,997154,997850,996393,995655,990167,998006,993948,993749,990423,990558,990154,994727,994467,993381,997030,993956,990361,994374,991836,994643,991385,991596\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 619\n", + "Returned IDs: 997995,997455,995095,996609,992281,997993,998221,990432,993219,995401,992619,991836,990435,993209,999820,990126,994587,995354,999923,992979,997185,995289,993412,999160,990157,993656,995633,995374,999240,998297\n", + "Ground Truth: 997995,997455,995095,996609,992281,997993,998221,992087,990432,993219,991445,995401,992619,991836,990435,996088,993209,999820,990126,994587,995354,999923,992979,997185,995289,993412,999160,990157,993656,995633\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 620\n", + "Returned IDs: 997652,997897,998428,992052,993621,998522,993171,990956,999002,999179,998250,995421,992028,992747,990409,997368,990626,994280,994765,995213,994743,994742,997494,991605,996328,996461,999587,999927,990364,993175\n", + "Ground Truth: 997652,997897,998428,992052,993621,998522,993171,990956,999002,999179,998250,995421,992028,992747,990409,997368,990626,994280,994765,995213,994743,997723,994742,997494,991605,996328,996461,999587,999927,990364\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 621\n", + "Returned IDs: 995941,997371,990645,993507,994757,993818,998246,991782,995367,995755,996890,997192,993005,994281,990994,997194,992053,990863,999374,991379,991059,993518,997984,995353,999247,990969,993739,999290,992978,991456\n", + "Ground Truth: 995941,997371,990645,993507,994757,993818,998246,991782,995367,995755,996890,997192,993005,994281,990994,997194,994183,992053,990863,999374,991379,993326,991059,993518,997984,995353,999247,997022,990969,993739\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 623\n", + "Returned IDs: 992558,999564,992738,995724,998827,994372,993140,997124,990273,993077,995983,991009,997158,999146,996918,990538,992170,992179,994163,991260,991905,991157,997775,995411,999639,994742,993706,992615,998142,997859\n", + "Ground Truth: 992558,999564,992738,995724,998827,994372,993140,997124,990273,993077,995983,991009,995281,997158,991585,999146,996918,990538,992170,992179,997411,994163,991260,991905,991157,997775,995411,999639,994742,995401\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 625\n", + "Returned IDs: 997977,992823,993504,995168,994468,991425,992082,997618,994672,996217,998303,997140,998007,995536,995231,993260,990196,997002,993071,997790,997918,995740,999111,996001,994955,995203,991863,996884,990350,997913\n", + "Ground Truth: 997977,992823,993504,995168,994468,991425,992082,997618,994672,996217,998303,997140,998007,995536,995231,993260,990196,997002,993071,997790,997918,995740,999111,996001,994955,995203,991863,995167,996884,990350\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 626\n", + "Returned IDs: 994906,993289,993342,999587,992664,993820,991459,995401,994946,992636,998855,998536,993275,990399,998623,999977,997332,996764,997002,996001,999820,999724,992179,996147,997910,994743,997790,993715,998831,999453\n", + "Ground Truth: 994906,993289,993342,999587,992664,993820,991459,995401,994946,992636,998855,998536,993275,990399,998623,999977,997332,996764,997002,996001,999820,999724,996371,992179,996147,997910,994743,997790,993715,998831\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 628\n", + "Returned IDs: 999457,990194,998279,999555,999994,999205,993461,990167,999223,998674,999430,992628,998072,993076,994109,993224,998586,998570,991038,997415,991610,995234,995848,991025,999985,998086,998078,995907,997506,994731\n", + "Ground Truth: 999457,990194,998279,999555,999994,999205,993461,990167,999223,998674,999430,992628,998072,993076,994109,993224,998586,998570,991038,997415,991610,995234,995848,991025,999985,998086,998078,995907,995082,997506\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 629\n", + "Returned IDs: 993250,995135,990898,998751,999649,998660,995865,991582,991172,991840,996854,994819,998011,998996,996352,995365,999676,998915,993640,997479,990965,990310,993253,995693,990979,991999,994415,998540,997192,994497\n", + "Ground Truth: 993250,995135,990898,998751,999649,998660,995865,991582,991172,991840,996854,994819,998011,998996,996352,997641,995365,999676,998915,997291,993640,997479,990965,990310,993253,995693,993404,990979,991999,994415\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 630\n", + "Returned IDs: 994852,992190,999451,997106,990884,999797,994622,996894,995138,998455,995752,992349,993789,997818,993355,999645,997076,990096,996884,995769,993706,998974,992668,996196,999807,999639,996262,990522,993222,991609\n", + "Ground Truth: 994852,992190,999451,997106,990884,999797,994622,996894,995138,998455,995752,992349,993789,997818,993355,999645,997076,990096,996884,995769,993706,998974,992668,996196,999807,991637,999639,992612,996262,990351\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 632\n", + "Returned IDs: 995210,996808,997918,996062,998436,993512,990366,993289,990891,997002,991425,994821,992711,990885,994468,994042,993260,994959,996001,993504,990622,995826,994694,991833,993268,999091,991311,992420,993157,994538\n", + "Ground Truth: 995210,996808,997918,996062,998436,993512,990366,993289,990891,997002,991425,997332,994821,992711,990885,994468,994042,993260,992612,994959,997531,996001,993504,990622,995826,998611,994694,991833,993268,999091\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 633\n", + "Returned IDs: 999854,990584,998228,993637,998436,997790,999827,996062,990557,995678,994472,997918,990091,997002,998039,999516,996001,993906,993765,997586,997920,992576,991994,992161,995107,993484,997453,992395,990080,998734\n", + "Ground Truth: 999854,990584,998228,993637,998436,997790,999827,996062,990557,995678,994472,997918,990091,997002,998039,993238,999516,996001,993906,993765,995502,997586,992576,997920,991994,992161,995107,993484,997453,992395\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 636\n", + "Returned IDs: 995557,999604,996596,990310,993748,999592,998899,993237,995365,991593,993279,996748,995588,993091,992207,998917,995952,993675,992195,994856,998997,998057,999039,998654,994263,992085,995072,991519,994964,997275\n", + "Ground Truth: 995557,999604,996596,990310,993748,999592,998899,993237,995365,991593,993279,996748,995588,993091,992207,998917,995952,995794,993675,992195,994856,998997,998057,999039,998654,994263,992085,995072,994005,991519\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 638\n", + "Returned IDs: 999638,990926,992349,992718,997816,997859,994163,991540,996894,995138,996318,994010,991498,991977,998697,998417,997113,998118,998334,999387,994964,992124,994165,990908,990343,995536,994972,991637,999908,995986\n", + "Ground Truth: 999638,990926,992349,992718,997816,997859,994163,991540,993948,996894,995138,996318,994010,991498,991977,998697,998417,997113,998118,998334,999387,994964,992124,994165,990908,990343,995536,994972,991637,999908\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 639\n", + "Returned IDs: 994137,999358,995378,990560,990390,999363,991900,998462,995942,998966,992310,991396,993403,990146,993072,991444,997972,993484,999229,990742,994964,997880,994479,999110,997539,999710,990311,996629,998365,995949\n", + "Ground Truth: 994137,999358,995378,990560,990390,999363,991900,998462,995942,998966,992310,991396,993403,990146,993072,991444,997972,993484,991178,999229,990742,994964,997880,994479,999110,993284,999390,997539,999710,990311\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 640\n", + "Returned IDs: 996127,993839,998220,999860,999639,993561,991923,990550,990303,994117,995013,991958,996181,996196,998593,991332,993489,990167,996391,994372,999393,991350,996379,995562,994796,998186,995983,998514,995855,998697\n", + "Ground Truth: 996127,993839,998220,999860,999639,993561,991923,990550,990303,994117,995013,991958,992473,996181,996196,998593,991332,993489,990167,996391,994372,999393,991350,991212,996379,995562,994796,998186,995983,998514\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 641\n", + "Returned IDs: 991810,996809,990127,993817,999788,994682,994742,992827,998063,999183,992372,997349,997319,993744,995633,999852,990223,995222,995895,990173,995401,998536,998495,990986,990722,996408,997401,990392,991620,998525\n", + "Ground Truth: 991810,996809,990127,993817,999788,994682,994742,992827,998063,999183,992372,997349,997319,993744,995633,999852,990223,995222,992318,995895,990173,995401,998536,998495,990986,990722,996408,997401,990392,997529\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 643\n", + "Returned IDs: 997749,999411,994909,995421,994765,993727,992934,998123,990521,994280,991491,996276,990844,994541,993678,998934,995963,997897,990998,995110,994697,998510,994809,992112,990021,993448,999601,992457,999179,995401\n", + "Ground Truth: 997749,999411,994909,995421,994765,993727,992934,998123,990521,994280,999453,991491,996276,990844,994541,993678,998934,995963,997897,990998,995110,994697,998510,994809,992112,990021,993448,999601,992457,999179\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 644\n", + "Returned IDs: 996296,991456,990176,997022,994842,996075,990742,997699,992931,992880,997178,990957,997918,991131,990949,995417,998540,993253,996592,993696,998967,992442,991634,995336,992942,993529,994100,999710,992277,992978\n", + "Ground Truth: 996296,991456,990176,997022,994842,996075,990742,997699,992931,992880,997178,990957,997918,991131,990949,995417,998540,993253,996592,993696,998967,992442,991634,995336,993529,992942,994100,999710,992277,995981\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 645\n", + "Returned IDs: 997867,991740,993839,992376,998357,991977,993583,998879,995965,990908,993647,994614,991064,999008,993975,990894,991225,995491,997547,996663,992333,995459,993681,996585,999022,994254,993500,994398,999763,999387\n", + "Ground Truth: 997867,991740,993839,992376,998357,991977,993583,998879,995965,990908,993647,997407,994614,991064,999008,993975,990894,991225,995491,997547,996663,992333,995459,993681,996585,999022,994254,993500,994398,999763\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 646\n", + "Returned IDs: 997643,991309,994660,992638,991797,998277,994393,992943,997683,993566,993749,994294,997568,993666,999156,997110,994656,997030,992926,995044,997429,996393,990554,995994,997114,998888,995253,990132,991937,998858\n", + "Ground Truth: 997643,991309,994660,992638,991797,998277,994393,992943,997683,999530,993566,993749,994294,997568,993666,999156,997110,994656,997030,992926,995044,997429,996393,990554,995994,997114,998888,995253,990132,991937\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 647\n", + "Returned IDs: 990679,994552,998734,995855,995250,999473,993154,997370,994789,995203,999954,994212,999898,996075,996623,997918,993243,990352,993484,990652,994829,990470,993646,999876,999384,999132,997920,990957,992548,996001\n", + "Ground Truth: 990679,994552,998734,995855,995250,999473,993154,997370,994789,995203,999954,994212,999898,996075,996623,997918,993243,990352,993484,990652,994829,990470,993646,990809,996452,999876,999807,999384,999132,997920\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 648\n", + "Returned IDs: 994125,990641,996609,990655,992604,997918,995401,997403,991844,995289,998913,991996,996809,993412,994317,995718,993433,996411,996196,991179,996075,993725,995274,990126,997906,998999,994603,993561,998495,995881\n", + "Ground Truth: 994125,990641,996609,990655,992604,997918,995401,998383,997403,991844,998913,995289,991996,996809,993529,993412,994317,995718,993433,996411,996196,991179,996075,993725,995274,990126,997906,998999,994603,993561\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 649\n", + "Returned IDs: 996360,998045,993561,998545,991052,991958,993433,996042,995702,996322,994409,997822,991923,993412,996867,994419,996769,995679,995648,994549,994280,997868,997386,995873,998564,995536,997674,997675,997289,990137\n", + "Ground Truth: 996360,998045,993561,998545,991052,991958,999809,993433,996042,995702,996322,994409,990882,997822,991923,993412,996867,994419,996769,995679,995648,994549,994280,997868,997386,995873,998564,995536,997674,997675\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 650\n", + "Returned IDs: 993839,996127,996658,991432,998372,993718,995340,990061,990487,991028,998079,997864,991923,993976,998370,991743,991350,996391,992813,990638,995691,994514,995515,993583,998779,998879,997914,998686,992929,990602\n", + "Ground Truth: 993839,996127,996658,991432,998372,993718,995340,990061,990487,991028,998079,997864,991923,993976,998370,991743,991350,996391,992813,990638,995691,995515,994514,993583,998779,998879,997914,998686,995453,992929\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 652\n", + "Returned IDs: 999328,996646,997320,999647,997913,994450,993253,999109,997470,999948,991743,999324,998967,998550,998287,990182,994192,990650,996296,992969,999390,993387,998302,997826,999701,994266,998644,991593,993031,991616\n", + "Ground Truth: 999328,996646,997320,999647,997913,994450,993253,999109,997470,999948,991743,999324,998967,998550,998287,990182,994192,990650,997921,996296,992950,992969,999390,999258,993387,998302,997826,999701,994266,998644\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 654\n", + "Returned IDs: 995374,993636,997192,993852,997571,998844,994281,999374,996554,994850,999953,995833,990275,992147,992183,994218,990432,992309,998011,998263,995354,999144,993529,995336,999317,993443,999883,999804,992854,996075\n", + "Ground Truth: 995374,993636,997192,993852,997571,998844,994281,999374,996554,994850,999953,995833,990275,992147,992183,994218,990432,992309,998011,998263,995354,999144,993529,995336,999317,993443,999883,999562,999804,992854\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 655\n", + "Returned IDs: 996247,991459,999804,990294,999639,995894,997918,991979,994125,996407,991630,996500,997957,998022,992748,992473,993948,996918,992516,992429,995932,990897,992558,993188,999306,991657,992293,995701,995336,998758\n", + "Ground Truth: 996247,991459,999804,990294,999639,995894,997918,991979,994125,996407,995132,991630,996500,997957,994911,998022,992748,992473,993948,996918,992516,992429,995932,990897,992927,992558,993188,999306,993906,991657\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 656\n", + "Returned IDs: 999544,999676,992348,999453,994800,991787,999452,995968,992612,994195,993683,991673,996309,992907,992309,992739,990080,993781,991485,995372,999686,998607,992298,999254,994964,993633,991999,995365,999951,997498\n", + "Ground Truth: 999544,999676,992348,999453,994800,991787,999452,995968,992612,994195,993683,991673,996309,992907,992309,992739,990080,993781,991485,995372,996352,999686,998607,992298,999254,994964,993633,991999,995365,999951\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 658\n", + "Returned IDs: 994488,994416,992318,999060,993772,996809,999179,997332,990843,998950,991650,994541,997679,996001,993603,998543,994955,998218,993347,993721,991142,990930,995899,995678,991409,999320,996024,993293,997442,999009\n", + "Ground Truth: 994488,994416,992318,999060,993772,996809,999179,997332,990843,998950,991650,994541,997679,996001,993603,998543,994955,998218,993347,993721,991142,990930,993079,995899,995678,991409,995167,999320,996503,996024\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 661\n", + "Returned IDs: 993461,990479,996918,991052,996535,990897,998267,993014,999646,997723,997586,990041,991758,998564,992933,998518,999980,995963,997910,994282,992979,997675,997216,991891,993412,992068,991666,992558,990942,998067\n", + "Ground Truth: 993461,990479,996918,991052,996535,990897,998267,993014,999646,997723,997586,990041,991758,998564,992933,998518,999980,995963,997910,998924,994282,992979,997675,997216,991891,998105,993412,992068,991666,992558\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 665\n", + "Returned IDs: 997777,990949,991593,997320,999290,993484,997370,992839,993253,999855,999535,990957,997539,995515,997372,995203,999453,990742,990890,994486,995942,999982,995400,999680,994826,992069,993389,999437,995596,993278\n", + "Ground Truth: 997777,990949,991593,997320,999290,993484,997370,992839,993253,999855,999535,990957,997539,995515,997372,995203,999453,990742,990890,996002,994486,995942,999982,995400,999680,994826,992069,993389,999437,995596\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 666\n", + "Returned IDs: 994954,992521,993438,994409,999866,992448,995640,995873,996368,997868,993561,996987,991969,995820,998255,996187,997091,990005,995701,998205,999250,997466,991349,998045,992426,991967,997723,997554,992937,997612\n", + "Ground Truth: 994954,992521,993438,994409,999866,992448,995640,995873,996368,996769,997868,993561,996987,991969,995820,998255,996187,997091,990005,995701,998205,999250,997466,991349,998045,992426,991967,997723,997554,993517\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 668\n", + "Returned IDs: 999117,993122,994467,995625,995044,991188,994643,997149,997154,992809,995207,999555,990991,999155,996395,992945,993795,999663,995257,993469,993995,993232,998861,996209,996393,992638,998536,998006,991385,997205\n", + "Ground Truth: 999117,993122,994467,995625,995044,991188,994643,997149,997154,992809,995207,999555,990991,999155,996395,992945,993795,999663,995257,993469,993995,993232,998861,996209,996393,992638,998536,995401,998006,999431\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 669\n", + "Returned IDs: 996489,990442,996981,996186,990249,990256,990209,992558,994334,992017,995332,997380,991426,996811,995552,990538,997155,995449,992179,999860,993473,995401,992170,999859,996196,993515,990620,995528,999715,999908\n", + "Ground Truth: 999901,996489,990442,996981,996186,990249,990256,990209,992558,994334,992017,995332,997380,991426,996811,995552,999480,990538,997155,995449,992179,999860,993473,995401,992170,998213,999859,996196,993515,990620\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 670\n", + "Returned IDs: 998289,998104,993324,992558,994732,990264,998950,996140,995289,996810,990769,994697,993079,997752,998697,996194,992988,995783,995281,996210,996370,994146,995359,994817,993018,996276,990956,997723,992052,997713\n", + "Ground Truth: 998289,998104,993324,992558,994732,990264,998950,996140,995289,996810,990769,995928,994697,993079,997752,994514,997630,998697,996852,996194,992988,995963,995783,995281,996210,996370,994146,995359,994817,993018\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 671\n", + "Returned IDs: 991925,996811,991743,998334,992702,993721,994119,990585,996918,993229,991977,999601,995375,996318,994662,998495,997481,999453,998849,992944,995873,999711,999707,995130,990640,996219,997436,996001,993121,991807\n", + "Ground Truth: 991925,996811,991743,998334,992702,993721,994119,990585,996918,993229,991977,999601,995375,996318,994662,998495,997481,999453,998849,994081,992944,997732,995873,999711,999707,995130,990640,996219,997436,996001\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 673\n", + "Returned IDs: 999703,992640,993314,993895,991161,993580,992728,990479,999520,998463,992598,994672,994280,991731,998045,990942,990882,994844,993315,998812,993609,995815,999441,996867,999052,999373,997192,994920,992752,995196\n", + "Ground Truth: 999703,992640,993314,993895,991161,993580,992728,990479,999520,998463,992598,994672,993433,994280,991731,998045,990942,990882,994844,993315,998812,993609,995815,999441,992617,996867,999052,999373,997192,994920\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 674\n", + "Returned IDs: 996913,994136,996761,996067,997839,997790,990550,996089,998473,999363,993834,998934,993766,990897,998185,993478,994910,992645,992429,992750,996795,990139,999216,999041,996547,993809,999384,993143,994915,996611\n", + "Ground Truth: 996913,994136,996761,996067,997839,996144,997790,990550,996089,998473,999363,993834,998399,998934,993766,990897,998185,993478,994910,992645,992429,992750,996795,990139,999216,999041,996547,993809,999384,996803\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 676\n", + "Returned IDs: 999290,997370,996840,992944,993484,998734,996762,995886,993143,999367,996949,996438,993052,992558,993080,995203,991840,990334,999426,991348,990697,998455,997839,990580,995596,993504,996397,992169,999453,993630\n", + "Ground Truth: 999290,997370,996840,992944,993484,998734,996762,995886,993143,999367,996949,996438,993052,992558,993080,995203,991840,990334,999426,991348,990697,998455,990351,997839,990580,995596,993504,996397,992169,999453\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 678\n", + "Returned IDs: 997348,999647,991582,994450,991428,996805,997913,993853,998426,991550,990893,999444,997918,993264,990231,994964,990797,997320,997880,996041,998921,998029,998550,990271,997301,993529,997586,994503,996809,994266\n", + "Ground Truth: 997348,999647,996904,991582,994450,991428,996805,997913,993853,998426,993141,991550,990893,997918,999444,993264,990231,994964,993412,990797,997320,997880,996041,996296,998921,998029,997772,998550,991721,999683\n", + "Recall: 0.7667\n", + "\n", + "Query ID: 680\n", + "Returned IDs: 997920,996232,996219,993412,994894,998649,999144,995274,991029,992337,999343,994757,996052,999261,999311,996504,995881,996001,995502,994819,999026,993168,993529,990949,999647,997479,995200,990975,990934,996869\n", + "Ground Truth: 997920,996232,996219,993412,994894,998649,999144,995274,991029,992337,999343,994757,996052,999261,999311,996504,991445,995881,995401,996001,995502,994819,999026,993168,993529,990949,999647,997479,995200,990975\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 681\n", + "Returned IDs: 997975,999137,999415,998188,993209,991671,993586,995778,994730,996302,991639,997723,999581,991507,996226,992289,999875,997544,993412,999054,996867,996852,995268,996187,990361,999957,998910,999980,992448,994742\n", + "Ground Truth: 997975,999137,999415,998188,993209,991671,993586,995778,994730,996302,991639,997723,999581,991507,996226,992289,999875,997544,993412,999054,996867,996852,995268,996607,996187,990361,999957,998910,999980,995963\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 682\n", + "Returned IDs: 998104,994300,993799,998536,993609,995963,990392,994119,990879,992617,994711,998568,992521,993324,996760,992131,997723,997752,995783,994409,999129,998162,996987,996328,997309,994280,997420,999886,996867,995401\n", + "Ground Truth: 998104,994300,993799,998536,993609,995963,990392,994119,990879,992617,994711,998568,992521,993324,999840,996760,992131,997723,997752,992289,995783,994409,999129,998162,996987,996328,997309,994280,997420,999886\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 688\n", + "Returned IDs: 999812,993504,993024,996429,994964,990818,999339,993260,999111,995895,998303,998715,998950,993391,996216,992747,997586,992571,993805,994672,995622,999735,990409,997977,991458,993489,996196,995887,991946,990494\n", + "Ground Truth: 999812,993504,993024,996429,994964,990818,999339,993260,999111,995895,998303,998715,998950,993391,996216,992747,997586,992571,996677,993805,992979,994672,995622,999735,990409,997977,991458,993489,996196,995887\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 689\n", + "Returned IDs: 992493,997634,995163,992677,998316,992150,992148,993223,997824,998650,995854,994620,993263,995723,992700,995060,991067,993530,995091,997750,996139,991190,996273,995062,991100,998080,999961,993160,991052,997723\n", + "Ground Truth: 992493,997634,995163,992677,998316,992150,992148,993223,997824,998650,995854,994620,993263,995723,992700,995060,991067,995963,993530,995091,997750,996139,991190,996273,995062,991100,998080,999961,993160,991052\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 690\n", + "Returned IDs: 992933,998536,999759,994835,992166,999259,993014,995175,996877,994119,990103,999533,995783,991052,994742,999847,994522,999244,995963,997309,997184,991712,994280,998021,991891,995685,997723,990184,995110,991747\n", + "Ground Truth: 992933,998536,999759,994835,992166,999259,993014,995175,996877,994119,990103,999533,995783,991052,994742,999847,994522,999244,995963,997309,999527,997184,991712,994280,998021,991891,995685,997723,990184,995110\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 691\n", + "Returned IDs: 999045,993721,994718,992551,990096,991409,998495,994269,991650,993053,996001,999336,996075,990843,996936,995361,994355,990584,991382,996219,999577,996631,998803,997550,991451,990091,996101,999343,994472,996016\n", + "Ground Truth: 999045,993721,994718,992551,990096,991409,998495,994269,991650,993053,996001,999336,996075,990843,996936,995361,994355,990584,991382,996219,999577,996631,998803,997550,991451,999302,990091,996101,999343,994472\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 694\n", + "Returned IDs: 990010,991425,999794,999211,993504,990956,993665,991552,998615,995349,992457,998303,999111,992137,991648,994672,996884,995541,992110,995103,992351,990752,997506,991037,996012,990658,993609,990409,994681,991257\n", + "Ground Truth: 990010,991425,999794,999211,993504,990956,993665,991552,998615,995349,992457,998303,999111,992137,991648,994672,998109,996884,995541,992110,995103,992351,990752,997506,991037,996012,990658,993609,990409,994681\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 696\n", + "Returned IDs: 993003,992267,996256,992917,994409,997466,991357,999250,995401,992448,997909,990295,992763,995242,996815,994508,994865,998910,998278,994106,993537,992521,997299,996968,998756,991197,995268,992260,999530,995864\n", + "Ground Truth: 993003,992267,996256,992917,994409,997466,999250,991357,995401,992448,997909,990295,992763,995242,996815,994508,994865,998910,998278,994106,993537,992521,996223,997299,996968,998756,991197,995268,992260,999530\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 697\n", + "Returned IDs: 993512,995336,997571,993096,999144,999331,998090,990647,992134,993853,992384,994197,990214,994810,992692,994334,993492,991131,993153,995357,995374,994819,993925,996882,998562,993785,998410,996296,992453,997876\n", + "Ground Truth: 993512,995336,997571,993096,999144,999331,998090,992134,990647,993853,992384,994197,990214,994810,992692,995748,994334,993492,991131,993153,995357,995374,994819,993925,991923,996882,990654,998562,993785,998410\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 700\n", + "Returned IDs: 990956,996016,991382,992652,992827,993501,994927,998543,994541,998495,996809,994378,995191,999179,994765,995895,993460,998167,996830,997270,991469,996706,999830,993219,996301,998206,993948,990712,993391,997506\n", + "Ground Truth: 990956,996016,991382,992652,992827,993501,994927,998543,994541,998495,996809,994378,995191,999179,994765,995895,993460,998167,996830,997270,995401,991469,996706,999830,993219,996301,998206,993948,990712,993391\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 701\n", + "Returned IDs: 990930,991459,992036,992790,999639,998874,995536,995401,996001,994987,990412,993515,993673,996247,995932,991408,991530,997918,998990,995672,998593,991052,996918,998303,991392,996181,997238,991721,994280,992749\n", + "Ground Truth: 990930,991459,992036,992790,999639,998874,995536,995401,996001,994987,990412,993515,993673,990337,996247,995932,991408,996371,991530,997918,998990,995672,998593,991052,996918,998303,991392,996181,997238,991721\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 704\n", + "Returned IDs: 990003,997913,999647,990210,990545,996053,998344,999331,994873,998302,990958,993031,991479,990231,998540,991429,995438,990650,991852,991782,994450,993484,997320,993990,996296,998550,992789,999167,999263,999109\n", + "Ground Truth: 990003,997913,999647,990210,990545,992946,996053,998344,999331,994873,998302,990958,993031,991479,997439,990231,998540,991429,995438,990650,991852,991782,994450,993484,997320,993990,991370,994898,996296,998550\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 705\n", + "Returned IDs: 997897,994280,995421,991605,991447,991232,993489,992052,990627,997442,990844,992864,997926,998071,995626,995203,992457,991332,996817,997531,995630,998428,990885,997790,991218,991212,993175,999550,993391,994738\n", + "Ground Truth: 997897,994280,995421,991605,991447,991232,993489,992052,990627,997442,990844,992864,997926,998071,995626,993087,995203,992457,991332,996817,997531,995630,998428,990885,997790,991218,991212,993175,999550,993391\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 706\n", + "Returned IDs: 993368,998798,995374,996703,990992,997015,991467,992854,996353,998951,990585,994197,991504,997609,990535,999026,996959,994595,997009,991173,991498,999089,996714,992189,996658,993562,995824,995034,998844,999024\n", + "Ground Truth: 993368,998798,995374,996703,990992,997015,991467,992854,996353,998951,990585,994197,991504,997609,990535,993909,999026,996959,994595,997009,991173,991498,999089,996714,992189,996658,993562,995824,995034,998844\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 708\n", + "Returned IDs: 996052,996219,998383,990448,999998,998464,991743,995716,994964,993412,991844,996232,995581,990538,999876,997010,992604,990063,991650,990366,996001,998593,999791,992944,992337,995107,992558,998495,990783,991052\n", + "Ground Truth: 996052,996219,998383,990448,999998,998464,991743,995716,991999,994964,993412,991844,996232,995581,990538,999876,997010,992604,990063,991650,990366,996001,998593,999791,995596,992944,992337,995107,992558,990091\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 710\n", + "Returned IDs: 994922,991840,997591,990958,999261,992169,996001,998593,997822,990076,996351,992994,999729,990648,990399,996867,999676,991087,990064,993143,992692,996854,997320,992944,990866,990455,994380,992622,996438,995401\n", + "Ground Truth: 994922,991840,997591,990958,999261,992169,996001,998593,997822,990076,996351,992994,999729,990648,990399,996867,999676,991087,990064,993143,992692,996854,997320,992944,990866,990455,997620,994380,992622,996438\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 712\n", + "Returned IDs: 991720,990545,996296,998170,993143,999437,999764,991999,993253,998562,994251,990432,999077,999851,994964,995588,995515,995927,997876,998294,997908,993484,999343,997320,998550,992384,993885,990504,993238,994100\n", + "Ground Truth: 991720,990545,996296,998170,993143,999437,999764,991999,993253,991715,998562,994251,992979,990432,999077,999851,994964,995588,995515,995927,997876,998294,997908,993484,999343,997320,998550,992384,995748,993885\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 713\n", + "Returned IDs: 999384,994972,997811,990096,995536,998697,991179,997320,999898,999647,996001,996062,999144,997913,991154,995701,996219,995130,993504,998401,993809,997775,993906,996370,992047,995596,993561,991459,995560,995401\n", + "Ground Truth: 999384,994972,997811,990096,995536,998697,991179,997320,999898,999647,996001,996062,999144,997913,991154,995701,996219,995130,993504,998401,993809,992692,997775,993906,996370,992047,995596,993561,991459,995560\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 714\n", + "Returned IDs: 992669,990063,997526,998858,992604,999961,997110,992439,993115,991309,998707,994724,994774,995982,993122,999478,990111,998383,990657,994569,999231,997205,991987,994454,997107,998806,995455,992051,991052,998297\n", + "Ground Truth: 992669,990063,997526,998858,992604,999961,997110,992439,993115,995949,991309,998707,994724,994774,995982,993122,999478,990111,998383,990657,994569,999231,997205,991987,994454,997107,993976,998806,995455,992051\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 718\n", + "Returned IDs: 996536,993588,994897,994830,993342,997952,992664,998716,993335,998092,995097,998623,998903,998440,997296,998256,997073,998030,990334,996183,992427,998097,994513,993866,990392,999963,992458,994682,992174,993715\n", + "Ground Truth: 996536,993588,994897,994830,993342,997952,992664,998716,997301,993335,998092,995097,998623,998903,998440,997296,998256,997073,998030,990334,996183,992427,998097,994513,993866,990392,999963,992458,994682,992174\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 719\n", + "Returned IDs: 992330,997371,997913,995985,993072,990397,990469,990560,993048,998500,991468,990092,992384,992443,997889,996193,990595,998921,995260,996840,994360,992546,994281,999122,995365,997320,999676,997984,997305,997831\n", + "Ground Truth: 992330,997371,997913,995985,993072,990397,990469,990560,993048,998500,991468,990092,992384,992443,997889,996193,990595,998921,995260,996840,994360,992546,994281,999613,991631,999122,995365,997320,999676,997984\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 720\n", + "Returned IDs: 998555,998169,999781,990271,990096,997529,998122,999343,995638,992551,993800,997732,990091,994145,992120,995535,999103,993637,996219,990089,999647,999319,990690,997918,994532,997920,998314,990918,996328,990072\n", + "Ground Truth: 998555,998169,999781,990271,990096,997529,998122,999343,995638,992551,993800,997732,990091,994145,992120,995535,999103,993637,996219,995216,990089,999647,999319,990690,997918,994532,998934,997920,998314,990918\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 724\n", + "Returned IDs: 993775,998765,999791,995477,991921,990089,994742,997529,999058,996633,997401,995057,997698,996328,996302,999788,997309,999875,991588,990179,999192,999261,995438,994806,993053,993265,995289,995551,995600,999886\n", + "Ground Truth: 993775,998765,999791,995477,991921,990089,994742,997529,999058,996633,997401,995057,997698,996328,996302,999788,997309,999875,991588,990179,999192,999261,995438,994806,993053,993265,995289,995551,999138,995600\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 726\n", + "Returned IDs: 993035,995791,995004,995365,993570,993633,995588,999039,999592,993282,998837,990043,999620,993028,991999,997496,995155,994545,995657,994666,991407,991593,993675,994543,994922,993708,993946,991196,990155,996402\n", + "Ground Truth: 993035,995791,995004,995365,993570,993633,995588,999039,999592,993282,998837,990043,999620,993028,991999,997496,995155,994545,995657,994666,991407,991593,993505,993675,994543,994922,993708,993946,991196,990155\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 727\n", + "Returned IDs: 995865,994611,996143,993745,996318,993154,991872,994214,997320,990210,997178,993579,994812,992909,996805,997022,993885,999351,998550,998287,997305,991498,990772,998344,994964,999374,994699,994249,996755,992978\n", + "Ground Truth: 993990,995865,990612,994670,994611,996143,993745,996318,993154,991872,994214,997320,990210,997178,993579,994812,992909,996805,997022,993885,999351,998550,998287,997305,991498,990772,998344,994964,999374,994699\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 728\n", + "Returned IDs: 995932,993504,999516,990196,993071,993906,991440,997758,991648,997555,991206,994081,997135,995622,993515,990682,992082,993821,990350,997918,992571,993783,997790,992747,993260,997002,992351,991435,997158,995375\n", + "Ground Truth: 995932,993504,999516,990196,993071,993906,991440,997758,991648,997555,991206,994081,997135,995622,993515,990682,992082,993821,990350,997918,992571,993783,997790,992747,993696,993260,997002,994146,992351,991435\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 730\n", + "Returned IDs: 997025,995521,995907,998703,999653,995303,991161,992448,997251,995891,995537,997909,993275,993313,993342,994380,990593,995778,990725,994125,991844,997723,991338,997675,993561,996042,993127,991459,991447,991794\n", + "Ground Truth: 997025,995521,995907,998703,999653,995303,991161,992448,997251,995891,995537,997909,993275,993313,993342,994380,990593,995778,990725,994125,991844,997723,991338,997675,993561,996042,993127,998855,991459,991447\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 731\n", + "Returned IDs: 999374,999710,997320,997913,999701,996504,992748,991860,999331,991801,998294,998495,998540,993484,995942,991743,997301,996178,992789,992147,995886,994964,993809,995961,994850,991498,998246,990958,990957,998550\n", + "Ground Truth: 999374,999710,997320,997913,999701,991052,996504,992748,991860,999331,991801,998294,998495,998540,993484,995942,991743,995632,997301,996178,992789,992147,995886,994964,993809,995961,994850,991498,998246,990958\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 733\n", + "Returned IDs: 992037,998022,996277,998934,994906,995401,996181,990400,996001,998720,991459,994444,999020,992685,999026,999543,997204,994105,995091,993652,994589,994424,992648,996379,992576,999759,998990,990453,991392,991542\n", + "Ground Truth: 992037,998022,996277,998934,994906,995401,993880,996181,990400,996001,998720,998080,991459,999259,994444,999020,992685,999026,998221,999543,997204,994105,995091,996408,993652,995780,994589,994424,992648,996379\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 734\n", + "Returned IDs: 992712,995855,995356,995563,998622,992163,996105,994167,990472,994955,993283,999216,997368,993300,998513,991232,996144,992484,999041,993174,999219,998697,997977,990688,999750,995672,998934,997417,995203,993821\n", + "Ground Truth: 992712,995855,995356,995563,998622,992163,996105,994167,990472,994955,993283,999216,997368,993300,998513,991232,996144,992484,999041,993174,999219,998697,997977,990688,999750,995672,998934,997417,995284,995203\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 735\n", + "Returned IDs: 996196,997775,998950,995873,994163,996140,994661,995596,997320,995560,990538,990532,998417,996178,991157,995983,990930,990096,992124,992277,990901,992558,991923,997913,997227,995552,997379,999564,996380,996721\n", + "Ground Truth: 996196,997775,998950,995873,994163,996140,994661,995596,997320,995560,990538,992521,990532,996178,998417,991157,995983,990930,990096,992124,992277,990901,992558,991923,997913,997227,995552,997379,995974,999587\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 739\n", + "Returned IDs: 992932,990535,997304,998246,995981,998119,999761,992296,991900,998353,992815,997320,995473,991045,995132,991534,992238,995792,990588,993392,993154,990949,997428,998225,997353,992074,998950,990057,992357,999357\n", + "Ground Truth: 992932,990535,997304,996732,998246,995981,998119,999761,992296,991900,998353,992815,997320,995473,991045,995132,991534,992238,995792,990588,993392,993154,996950,992927,990949,997428,998225,998361,997353,992074\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 740\n", + "Returned IDs: 992862,998156,993857,993256,994564,991428,997173,990328,997374,994742,996328,995819,997452,997309,999843,997918,995633,995295,998708,996752,997852,997914,990814,998073,992473,991189,996411,997409,995600,997208\n", + "Ground Truth: 992862,998156,993857,993256,994564,994940,991428,996232,997173,990328,993440,997374,992979,994742,996328,995819,997452,997309,999843,997918,995633,995295,994850,998708,996752,997852,997914,990814,999453,998073\n", + "Recall: 0.8000\n", + "\n", + "Query ID: 742\n", + "Returned IDs: 995231,997369,995168,994105,997666,992275,995385,990096,999176,996702,994779,995257,992987,993866,997965,990429,990658,999576,992945,997847,998536,995393,997086,998177,992664,999129,996050,994834,990670,991654\n", + "Ground Truth: 995231,997369,995168,994105,997666,992275,995385,990096,999176,996702,994779,995257,992987,993866,997965,990429,990658,999576,992945,991313,997847,998536,995393,997086,998177,992664,999129,996050,994834,990670\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 743\n", + "Returned IDs: 995121,991630,997191,991140,998297,992977,996734,993738,995401,990005,999530,992448,994319,998045,997868,998205,995864,991969,995395,994019,997957,991958,993381,998346,996277,999820,992521,999731,995940,997723\n", + "Ground Truth: 995121,991630,997191,991140,998297,992977,996734,995268,993738,995401,990005,999530,992448,994319,998045,997868,998205,995864,991969,995395,994019,997957,991958,993381,998346,996277,997928,999820,992521,999731\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 744\n", + "Returned IDs: 997006,993885,998294,997876,997320,990898,999374,996296,993364,990917,991059,991900,995266,992896,993048,993853,994964,994776,990760,994476,999328,996218,993484,990863,999647,999823,998246,992931,990560,990146\n", + "Ground Truth: 997006,993885,998294,997876,997320,990898,999374,996296,993364,990917,991059,991900,995266,992896,993048,993853,994964,994776,990760,994476,999328,996218,993484,999247,990863,998899,999647,999823,998246,990296\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 745\n", + "Returned IDs: 995793,991022,992518,995833,991782,990862,998822,990535,996713,995771,998951,997976,994266,997320,993735,994100,995336,998263,996714,998721,996961,992646,992183,992384,999089,995279,997571,991889,993711,992609\n", + "Ground Truth: 995793,991022,992518,995833,991782,990862,998822,990535,996713,995771,998951,997976,994266,997320,993735,994100,995336,998263,996714,998721,996961,992646,992183,992384,995438,999089,995279,997571,991889,999533\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 748\n", + "Returned IDs: 999453,991930,990772,996001,991248,991440,991609,993280,990314,995250,994484,992395,995968,997332,991787,992884,990399,991581,995372,990532,993032,994449,999575,991158,994478,991542,991994,992182,992469,998567\n", + "Ground Truth: 999453,991930,990772,996001,991248,991440,991609,993280,990314,995250,994484,992395,995968,997332,991787,998127,992884,990399,991581,995372,990532,993032,994449,999575,991158,994478,991542,991994,992182,992469\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 749\n", + "Returned IDs: 996111,997032,995482,994925,996232,998221,996328,994742,994798,999647,990096,993053,990435,994998,992450,996196,996178,998649,990887,998495,996001,991743,997379,999898,998067,996884,993783,991769,997126,998710\n", + "Ground Truth: 996111,997032,995482,994925,996232,998221,996328,994742,994798,999647,990096,993053,990435,994998,992450,996196,996178,998649,990887,998495,999586,996001,991743,997379,999898,998067,996884,993783,991769,997126\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 751\n", + "Returned IDs: 997015,991568,993168,993050,998992,990535,992744,995336,993562,996714,998951,995252,992646,999531,997082,994964,999026,990936,995824,992911,995536,992331,993648,999144,994197,999437,991444,997709,990925,993484\n", + "Ground Truth: 997015,991568,993168,993050,998992,990535,992744,995336,993562,996714,998951,995252,992646,999531,997082,994964,999026,990936,995824,992911,995536,992331,993648,999144,994197,998495,999437,991444,997709,990925\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 754\n", + "Returned IDs: 998950,996805,999647,992944,993696,992041,998180,997906,999210,999426,997470,991650,997371,995633,997403,997913,993504,994742,997291,995401,999676,998495,999639,993747,993529,998790,992520,996196,996219,990096\n", + "Ground Truth: 998950,996805,999647,992944,993696,992041,998180,997906,999210,999426,994089,997470,991650,997371,995633,997403,997913,993504,994742,997291,995401,999676,998495,998855,999639,993747,993529,998790,992520,996196\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 758\n", + "Returned IDs: 996001,991119,995455,995755,999009,998861,991881,997337,990505,998122,990096,996809,996406,994879,995701,990866,995425,999480,996408,998514,999388,997586,995401,990091,995132,991459,999289,991154,995536,997305\n", + "Ground Truth: 996001,991119,995455,995755,999009,998861,991881,999916,997337,990862,990505,998122,990096,996809,996406,994879,995701,990866,995425,999480,996408,998514,999388,997586,995401,990091,995132,991459,999289,991154\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 760\n", + "Returned IDs: 990061,990273,995309,993583,990206,994254,993500,996380,991064,997547,992052,990042,994679,993077,995459,999022,993811,991977,991432,997866,995728,993742,990776,996328,992558,990894,993879,994280,997494,991447\n", + "Ground Truth: 990061,990273,995309,993583,990206,994254,993500,996380,991064,997547,992052,990042,994679,993077,995459,999022,993811,991977,991432,997866,995728,993742,990776,996328,995600,992558,990894,993879,994280,997494\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 761\n", + "Returned IDs: 990127,995198,990331,993209,995873,995778,992862,994409,995268,995401,990887,999179,993313,994488,992260,990975,996328,994954,992293,990930,992052,994835,992636,992595,995122,999581,991958,995864,998623,993561\n", + "Ground Truth: 990127,995198,990331,993209,995873,995778,992862,994409,995268,995401,990887,995536,999179,993313,994488,992260,990975,997184,996328,994954,992293,990930,992052,994835,992636,993809,992595,995122,999581,991958\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 762\n", + "Returned IDs: 997550,995596,994618,998921,999338,999426,996075,997661,997113,991992,995033,992041,995515,997448,996318,993785,992944,997584,996516,994971,997984,991984,993125,994233,996741,996631,996387,999698,999588,995398\n", + "Ground Truth: 997550,995596,994618,998921,999338,999426,996075,997661,997113,991992,995033,992041,995515,997448,996318,993785,992944,997584,996516,994971,997984,991984,993125,995401,994233,996741,996631,996387,999698,999588\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 763\n", + "Returned IDs: 995317,993558,993678,996194,992992,997854,993256,994541,993044,990987,997251,990069,999287,997429,995159,996438,991863,994064,999386,997806,994428,993621,993014,992313,996232,991285,997770,994639,999179,990776\n", + "Ground Truth: 995317,993558,993678,996194,992992,997854,993256,994541,991075,993044,990987,997251,990069,995401,999287,997429,995159,996438,991863,994064,999386,997806,994428,993621,993014,992313,996232,991285,993454,997770\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 764\n", + "Returned IDs: 994955,998933,993194,999798,991685,993665,999052,998536,997944,992825,994280,995168,997155,992850,995932,992561,993715,995589,995366,997453,992823,998778,995945,993931,993037,994167,990387,999840,990364,999543\n", + "Ground Truth: 994955,998933,993194,999798,991685,993665,999052,998536,997944,992825,994280,995168,997155,992850,995932,992561,993715,995589,995366,997453,992823,998778,995945,993931,997926,993037,994167,990387,999840,990364\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 765\n", + "Returned IDs: 994557,991058,997638,997807,993929,994119,995332,990879,990345,992704,997822,997309,990987,993014,990678,999210,997693,996071,991107,999643,990226,992147,994280,997945,994336,997126,992962,994541,991177,991487\n", + "Ground Truth: 994557,991058,997638,997807,993929,994119,995332,990879,990345,992704,997822,997309,990987,993014,990678,999210,997693,996071,991107,995620,999643,990226,992147,994280,997945,994336,997126,992962,994541,991177\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 768\n", + "Returned IDs: 999117,994643,993122,993608,995625,992677,999155,993232,990361,991987,996395,998078,999231,991630,997149,995807,999834,991052,996809,993795,998257,997110,994092,992206,997752,997568,995373,998536,995105,992288\n", + "Ground Truth: 999117,994643,993122,993608,995625,992677,999155,993232,990361,996395,991987,998078,999231,991630,997149,995807,995401,999834,991052,996809,993795,998257,997110,994092,992206,997752,997568,995373,998536,995105\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 770\n", + "Returned IDs: 991889,995949,998294,995260,996053,998692,999374,991705,991406,996554,993837,992309,995928,993143,997627,996334,992979,995693,994160,994919,990432,996854,990490,995354,996202,992169,991445,997429,997083,994850\n", + "Ground Truth: 991889,995949,998294,995260,996053,998692,999374,991705,991406,996554,993837,992309,990026,995928,993143,997627,996334,992979,995693,994160,994919,990432,990545,990103,993412,996854,990490,995354,996202,992169\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 771\n", + "Returned IDs: 993416,992684,994855,991958,998158,998382,990794,993561,994987,996040,996062,990254,993673,991052,996371,999236,998593,992017,995923,994946,995547,993738,998323,994419,997453,999868,992260,998297,995373,998891\n", + "Ground Truth: 993416,992684,994855,991958,998158,998382,990794,993561,994987,996040,996062,990254,993673,991052,996371,999236,998593,992017,995923,994946,995547,993738,994076,996834,998323,994419,997453,999868,992260,998297\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 772\n", + "Returned IDs: 997002,998902,994906,992790,994334,997301,996658,999442,998951,995560,991621,994661,992239,996001,995401,992165,995374,995836,992759,996220,994362,993820,995357,998045,997918,990140,999587,994088,992641,995536\n", + "Ground Truth: 997002,998902,994906,992790,994334,995894,997301,996658,999442,998951,995560,991621,994661,992239,996001,995401,992165,995374,995836,992759,996220,994362,993820,995357,998045,997918,991989,990140,999587,994088\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 773\n", + "Returned IDs: 992041,996075,990022,990366,993785,992167,990697,991372,995515,994440,999426,991984,996001,991109,999473,997836,998654,997661,993928,995596,996227,992872,998498,996219,993484,992733,992927,999053,992617,999806\n", + "Ground Truth: 992041,996075,990022,990366,993785,992167,990697,991372,995515,994440,999426,991984,996001,991109,999473,997836,994776,998654,997661,993928,995596,996227,992872,998498,996219,993484,992733,992927,999053,990934\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 774\n", + "Returned IDs: 992383,990383,991550,990884,999387,996445,999451,990376,991923,994964,996894,995983,992824,996196,998600,998401,990538,993781,995427,995372,999639,993706,997903,994979,994852,995459,991817,993492,992190,996085\n", + "Ground Truth: 992383,990383,991550,990884,999387,996445,999451,990376,991923,994964,996894,995983,992824,996196,998600,998401,990538,993781,995427,995372,999639,993706,997903,994979,994852,995459,991817,993492,992190,998545\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 775\n", + "Returned IDs: 990832,995100,990194,993435,997725,997331,996475,996809,996219,996830,992892,999435,991187,995880,991459,995131,999331,999384,996067,996216,993342,998122,995130,999430,999388,999647,993192,996913,992100,997047\n", + "Ground Truth: 990832,995100,990194,993435,997725,997331,996475,996809,996219,996830,992892,999435,991187,995880,991459,995131,999331,999384,996067,996216,993342,998084,998122,995130,999430,999388,999647,993192,996913,998925\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 776\n", + "Returned IDs: 999159,991582,995632,997284,997354,994225,991348,996296,997320,992691,993018,991763,991992,993454,999777,995194,991879,996199,996387,991131,992896,990456,991662,996075,995108,996626,999658,994415,990375,998950\n", + "Ground Truth: 999159,991582,995632,997284,997354,994225,991348,995878,992022,996296,997320,992691,993018,991763,991992,993454,999777,995194,991879,996199,996387,991131,992896,990456,991662,996075,995108,996626,999658,994415\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 777\n", + "Returned IDs: 999663,995536,992225,998514,998861,999009,999711,990096,998720,991459,992293,996809,996001,992553,993139,994434,995401,995701,992862,998200,996406,999216,998082,998568,997586,993948,995961,997506,991893,998495\n", + "Ground Truth: 999663,995536,992225,998514,998861,999009,999711,990096,998720,991459,992293,996809,996001,992553,993139,994434,995401,993821,995701,992862,998200,996406,999216,998082,998568,997586,993948,995961,997506,991893\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 778\n", + "Returned IDs: 994401,993039,997077,998972,995065,992985,993931,995159,990184,998851,998566,996277,993799,994835,992576,997181,995760,995268,993256,996505,999759,990291,990942,995790,995815,993879,997126,990956,990391,995685\n", + "Ground Truth: 994401,993039,997077,998972,995065,992985,993931,995159,990184,998851,998566,996277,993799,994835,992576,997181,995648,995760,996223,995268,993256,996505,999759,990291,990942,995790,995815,993879,997126,990956\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 779\n", + "Returned IDs: 998496,995840,994879,996075,991166,992442,991984,993834,997258,992041,998708,993902,998200,997586,997305,993766,992330,996230,993809,996933,991476,995701,994558,996001,999701,996657,993696,999041,993283,991823\n", + "Ground Truth: 998496,995840,994879,996075,991166,992442,991984,993834,997258,992041,998708,993902,998200,997586,997305,997770,993766,992330,996230,993174,993809,996933,991476,995701,994558,996001,999701,996657,993696,999041\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 781\n", + "Returned IDs: 992226,994273,990815,994098,999430,998671,995704,999144,990711,993100,999435,996816,994558,997955,998416,993725,996222,994801,991672,999225,997331,996328,991938,999532,991378,998729,992951,993501,996856,993127\n", + "Ground Truth: 992226,994273,990815,994098,999430,998671,995704,999144,990711,993100,999435,996816,994558,991840,997955,998416,993725,993161,996222,994801,991672,999225,997331,996328,999171,998142,991938,999532,991378,998729\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 782\n", + "Returned IDs: 999530,998070,992448,995685,991301,994409,996323,991699,994002,996987,998714,998545,997046,997675,995507,992605,999578,999043,996322,993412,991671,996607,991247,999884,991583,993400,996042,995401,991052,995139\n", + "Ground Truth: 999530,998070,992448,995685,991301,994409,996323,991699,994002,996987,998714,998545,997046,997675,995507,992605,999578,999043,996322,993412,991671,996607,998697,991247,990337,999884,991583,993400,996042,995401\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 783\n", + "Returned IDs: 990089,995454,995720,997292,995659,999791,995045,993637,999858,992334,991537,999444,990918,996147,998076,998097,992636,998302,999302,995306,992415,996516,997490,994373,995133,991144,996001,995755,997349,997933\n", + "Ground Truth: 990089,995454,995720,997292,995659,999791,995045,993637,999858,992334,991537,999444,990918,996147,998076,998097,992636,998302,999302,995306,992415,996516,997490,994373,995133,991144,996001,995755,993663,997349\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 787\n", + "Returned IDs: 994670,991511,996219,992630,995274,990679,994916,992931,996714,990271,991085,992277,993154,994778,999243,997976,990910,995161,999340,997404,992748,999596,991456,995981,997018,995502,995901,990448,994894,990612\n", + "Ground Truth: 994670,991511,996219,993517,992630,995274,990679,994916,990909,992931,992124,999723,996714,990271,991085,992277,993154,994778,999243,997976,990910,995161,999340,997404,992748,999596,991456,995981,993481,997018\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 789\n", + "Returned IDs: 999240,993473,997195,993986,992558,999860,993811,991550,993077,995552,991025,992121,998929,990078,995972,999479,999715,996918,991817,996750,993158,993400,995967,996196,990249,994168,997054,991426,994679,992375\n", + "Ground Truth: 999240,993473,997195,993986,992558,999860,993811,991595,991550,993077,995552,991025,992121,998929,990078,995972,999479,999715,996918,991817,996750,993158,993400,995967,996196,990249,999631,994168,997054,991426\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 791\n", + "Returned IDs: 996633,992598,994639,996155,999059,999192,994541,998761,995551,990392,998348,993825,997026,994169,996960,992992,996194,997838,994356,993793,993448,993314,994742,993691,992832,992540,994327,993725,992595,994861\n", + "Ground Truth: 996633,992598,994639,996155,999059,999192,994541,998761,995551,990392,998348,993825,997026,994169,996960,992992,996194,997838,994356,993793,993448,993314,995401,994742,993691,992832,992540,994327,993725,992595\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 792\n", + "Returned IDs: 998902,998236,991131,998967,994604,991999,991359,990444,998001,993399,995515,991499,997918,998492,995357,998734,990176,999698,992839,991157,997831,992944,994329,995865,995988,995364,994789,994049,991609,991407\n", + "Ground Truth: 998902,998236,991131,998967,996030,994604,991999,991359,990444,998001,993399,995515,991499,997918,998492,995357,998734,990176,999698,992839,994449,991157,997831,992944,994329,990219,995865,995988,995364,994789\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 794\n", + "Returned IDs: 998679,990930,991585,993293,993813,998779,995633,995289,991141,994742,994730,994140,994436,993997,999402,998980,990435,993856,994409,991664,993831,995702,995648,997723,995216,997554,999805,995052,996277,999163\n", + "Ground Truth: 998679,990930,991585,993293,993813,998779,995633,995289,991141,994742,994730,994140,994436,993997,999402,998980,990435,993856,994409,991664,993831,995702,995648,997723,990361,995216,997554,999805,995052,996277\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 796\n", + "Returned IDs: 995252,992932,997320,997483,992096,990231,990541,997771,992228,995336,992277,992692,991900,992330,998090,992069,992969,998951,990885,998246,998365,998653,996296,993696,991131,990172,992978,995802,996196,992942\n", + "Ground Truth: 995252,992932,997320,997483,992096,990231,990541,997771,992228,995336,992277,992692,991900,992330,998090,992069,992969,998951,990885,998246,998365,998653,996296,993696,991131,990172,992978,995802,996196,996246\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 800\n", + "Returned IDs: 993708,995357,992134,993633,992692,997918,998967,993145,998996,991015,996592,991131,998011,999077,995336,995693,993492,993781,992946,990957,993512,998245,999144,996075,992277,993683,997470,999784,995210,996840\n", + "Ground Truth: 993708,995357,992134,993633,992692,997918,998967,993145,998996,991015,996592,993696,991131,998011,999077,995336,995693,993492,993781,992946,990957,993512,998245,999144,996075,992277,993683,997470,999784,995210\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 801\n", + "Returned IDs: 995203,993053,990918,993561,996645,990096,998416,995283,995873,990271,991464,994558,997370,994145,999653,994742,995310,997906,992334,998382,991168,994853,991565,999806,992408,995756,997920,990827,997305,999384\n", + "Ground Truth: 995203,993053,990918,993561,996645,990096,998416,995283,995873,990271,991464,994558,997370,994145,999653,994742,996147,993154,995310,997906,992334,998382,991168,994853,991565,999806,992408,995756,997920,990827\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 804\n", + "Returned IDs: 997309,997914,990705,992169,991720,998123,990776,999273,993677,994119,998495,996328,994765,994280,992364,996001,999179,995372,998180,990328,993993,991882,992944,991334,994310,994835,996809,994587,994531,997480\n", + "Ground Truth: 997309,997914,990705,992169,991720,998123,990776,999273,993677,994119,998495,996328,994765,994280,992364,996001,999179,995372,998180,990328,993993,991882,998466,994817,992944,991334,994310,994835,996809,994587\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 805\n", + "Returned IDs: 991642,990011,993906,995295,991435,993725,994879,995852,997002,994689,991315,998848,997790,992161,996811,990954,994206,994964,995823,992226,998359,996328,995586,998105,990494,997918,999500,994307,994012,994742\n", + "Ground Truth: 991642,990011,993906,995295,991435,993725,994879,995852,997002,994689,991315,998848,997790,992161,996811,990954,994206,994964,995823,992226,998359,991989,996328,995586,998105,990494,997918,999500,994307,994012\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 806\n", + "Returned IDs: 998343,990976,992056,995185,990255,990449,993875,997088,991256,995695,992293,998831,999334,995672,993545,994510,990399,998834,999498,998589,999663,998959,991836,999087,999502,997790,994815,995074,991038,992862\n", + "Ground Truth: 998343,990976,992056,995185,990255,990449,993875,997088,991256,995695,992293,998831,999334,995672,993545,994510,998453,990399,998834,999498,998589,999663,998959,991836,999087,999502,997790,996082,995074,994815\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 809\n", + "Returned IDs: 994964,996219,991582,993168,993289,999827,995927,997370,993143,997920,997822,999104,991052,997732,993412,997918,997586,994922,990866,994280,995210,990146,999343,992169,992147,997028,996001,995536,990382,990096\n", + "Ground Truth: 994964,996219,991582,993168,993289,999827,995927,997370,993143,997920,997822,999104,991052,997732,993412,997918,997586,994922,990866,994280,995210,990862,990146,999343,992169,992147,997028,999289,996001,995536\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 810\n", + "Returned IDs: 998314,999831,995385,993086,990096,998536,996918,992407,998177,998405,999287,993965,999079,994661,999194,992747,991052,996733,991093,998968,991724,991243,993239,999320,991571,993609,993715,997897,999384,990583\n", + "Ground Truth: 998314,999831,995385,993086,990096,998536,996918,992407,998177,998405,999287,993965,999079,994661,999194,992747,991052,996733,991093,998968,991724,991243,993239,999320,991571,993609,997775,993715,997897,999384\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 816\n", + "Returned IDs: 993392,992027,995771,993620,997984,996504,998921,996218,993518,997704,994281,994939,991782,998692,993457,995985,991873,994922,998822,992709,991889,999374,990595,997192,997006,992546,994775,992147,993161,994808\n", + "Ground Truth: 993392,992027,995771,993620,997984,996504,998921,996218,993518,997704,994281,994939,991782,998692,993457,995985,991873,994922,998822,992709,991889,992932,999374,990595,997192,997006,992546,994775,992147,993161\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 817\n", + "Returned IDs: 994304,997868,997586,992521,995814,994636,994954,996918,992293,997723,993438,997466,998260,995701,990866,999731,991790,998255,993831,997305,998346,995873,990097,991967,990078,996360,996187,994419,998934,997216\n", + "Ground Truth: 994304,997868,997586,992521,995814,994636,994954,996918,992293,997723,993438,997466,998260,995701,990866,999731,991790,996386,998255,993831,997305,998346,995873,990097,991967,990078,996360,996187,994419,998564\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 818\n", + "Returned IDs: 992909,994214,997320,998808,998090,992978,994290,991479,999621,990588,994427,991883,995372,995802,998364,992692,995826,995961,995228,994775,993662,992069,991503,991873,992969,994280,999761,997913,990910,993512\n", + "Ground Truth: 992909,994214,995090,997320,998808,998090,992978,994290,991479,999621,990588,994427,991883,995372,995802,998364,992692,995826,995961,995228,994775,993662,992069,991503,991873,992969,994280,999761,997913,998821\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 820\n", + "Returned IDs: 997331,998147,994113,995382,992433,993281,998770,999643,991011,996494,996722,993778,993939,996067,996866,995620,992840,991148,998484,990527,996219,991179,994692,995401,997837,996080,991377,994589,998855,994557\n", + "Ground Truth: 997331,998147,994113,995382,992433,993281,998770,996702,999643,991011,996494,996722,993778,993939,996067,996866,995620,992840,991148,998484,990527,996219,991179,994692,994984,995401,992987,997837,996080,991377\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 821\n", + "Returned IDs: 999837,992752,991633,998594,991242,991552,991605,998279,997673,993611,998389,997151,995167,996082,995421,998108,992864,990950,993715,999139,997050,995705,995626,990017,994734,996216,995232,994196,993347,995103\n", + "Ground Truth: 999837,992752,991633,998594,991242,991552,990133,991605,998279,997673,993611,998389,997151,995167,996082,995421,998108,992864,990950,993715,998716,999139,997050,995705,995626,990017,996216,994734,995232,994196\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 822\n", + "Returned IDs: 991765,993319,999798,996914,997340,996535,995768,998525,990408,997714,998168,994356,999601,990183,991779,997925,995222,992250,995501,992517,994841,992131,993993,996918,995755,990711,999886,999052,999496,998224\n", + "Ground Truth: 991765,993319,999798,993396,996914,997340,996535,995768,998525,990408,997714,998168,994356,999601,990183,991779,997925,995222,992250,993817,995501,992517,994841,992131,993993,996918,995755,990711,999886,999052\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 823\n", + "Returned IDs: 991356,992558,991977,997547,993255,990602,990226,994560,999908,993583,990894,999305,993077,997859,992113,993742,998151,992827,993839,994472,998892,995374,996318,995886,995552,995542,999026,990279,997918,998495\n", + "Ground Truth: 991356,992558,991977,997547,993255,990602,990226,994560,999908,993583,997806,990894,999305,993077,997859,992113,993742,998151,992827,993839,994472,990995,998892,995374,996318,995886,995552,995542,999026,990279\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 827\n", + "Returned IDs: 994051,993396,996899,999601,991399,994801,998766,997714,991052,996042,995963,996918,995342,997723,990189,994280,998123,993578,993805,993727,992933,996315,999820,996127,998267,993014,998778,993165,998397,990103\n", + "Ground Truth: 994051,993396,996899,999601,991399,994801,998766,997714,991052,996042,995963,996918,995342,997723,990189,994280,992569,998123,999596,993578,993805,993727,992933,996315,999820,996127,997803,995592,998267,993014\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 828\n", + "Returned IDs: 997116,996495,994142,999273,997506,998456,993139,996328,990712,995571,996408,993993,994680,990408,994196,998168,999985,991667,990091,999139,996431,991633,994012,993560,990705,994296,991552,990479,990449,998594\n", + "Ground Truth: 997116,996495,994142,999273,997506,998456,993139,996328,990712,995571,996408,993993,994680,990408,994196,990293,998168,999985,991667,990091,999139,996431,991633,994012,993560,990705,994296,991552,990479,990449\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 829\n", + "Returned IDs: 990310,993143,998033,991157,996840,999257,991392,995584,999122,999144,991609,991015,991840,995515,990580,999676,996829,997593,992950,992734,997379,993355,997320,996007,999546,997037,997918,998607,996282,992612\n", + "Ground Truth: 990310,993143,998033,991157,996840,999257,991392,995584,999122,999144,991609,991015,991840,995515,990580,999676,996829,997593,992950,992734,997379,998909,993355,997320,996007,994757,999546,992979,997037,997918\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 830\n", + "Returned IDs: 993324,995375,997184,990930,995130,990646,994092,991712,992933,993533,998272,998349,997675,996277,996328,993561,996591,991052,994743,993293,994409,992790,995080,993931,993347,999533,997309,993738,995401,995408\n", + "Ground Truth: 993324,995375,997184,990930,995130,990646,994092,991712,992933,993533,998272,998349,997522,996277,997675,996328,993561,996591,991052,994743,993293,996809,994409,992790,995080,993931,993347,999533,997309,993738\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 832\n", + "Returned IDs: 990934,995471,997529,996219,996507,999876,990974,996328,999343,995384,991335,996075,995107,996727,991743,990052,998668,999898,995096,992147,990866,998500,999144,995455,999374,997381,999417,999261,993412,991565\n", + "Ground Truth: 990934,995471,997529,996219,996507,999876,990974,996328,999343,995384,991335,996075,995107,996727,991743,990052,998668,999898,995096,992147,990866,999784,998500,999144,995455,995536,999374,991096,993335,997381\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 833\n", + "Returned IDs: 996339,998410,992582,996310,998011,997645,996220,992989,990284,998521,992558,992134,994819,997942,999820,995811,999400,997745,991389,996903,993296,990054,990921,994804,992428,999144,991680,995968,995886,997209\n", + "Ground Truth: 996339,998410,992582,996310,998011,997645,996220,992989,990284,992738,999240,998521,992558,992134,994819,997942,999820,995811,999400,997745,991389,996903,993296,990054,990921,994804,992428,999144,991680,992638\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 835\n", + "Returned IDs: 997301,997320,995274,998302,990231,995961,997913,999338,999109,997470,998779,990585,990949,991498,991743,995045,993484,990957,999533,990096,999290,992509,999710,994831,999801,994964,999647,999026,993431,998950\n", + "Ground Truth: 997301,997320,995274,998302,990231,995961,997913,999338,999109,997470,998779,990585,990949,991498,991743,995045,993484,990957,999533,990096,999290,992509,999710,994831,999801,999432,994964,999647,999026,993431\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 839\n", + "Returned IDs: 999216,992068,994988,995560,990786,992423,994638,996262,999806,999639,999711,990167,998383,992617,994852,993504,990769,996809,996053,999210,998292,993617,991157,998708,997864,994964,992282,991995,993355,990675\n", + "Ground Truth: 999216,992068,994988,995560,990786,994638,992423,996262,999806,999639,999711,990957,990167,998383,992617,994852,993504,994598,994660,990769,996809,996053,996857,999210,998292,993617,991157,998708,992979,997864\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 841\n", + "Returned IDs: 995443,990648,994046,998097,998726,998550,995080,998623,992933,993143,996292,999784,992636,999122,991999,998950,990257,992318,990096,995806,993719,999676,991801,992944,999210,993917,999701,998180,993260,997309\n", + "Ground Truth: 995443,990648,994046,998097,998726,998550,995080,998623,992933,993143,996292,999784,992636,999122,991999,998950,990257,993512,992318,990096,995806,993719,999676,994835,991801,992944,999210,993917,999701,998180\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 842\n", + "Returned IDs: 990670,999334,993535,993715,998468,996594,996050,998536,995746,999373,991436,995655,999711,992458,997685,996989,990293,992664,992933,997617,995969,995385,994723,991654,992110,990622,993917,992157,990942,995393\n", + "Ground Truth: 990670,999334,993535,993715,998468,996594,996050,998536,995746,999373,991436,995655,999711,992458,997685,996989,990293,992664,992933,997617,995969,995385,994723,991654,992110,990622,993917,992157,990942,997470\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 844\n", + "Returned IDs: 991470,996760,991257,997506,997019,998536,993381,995168,995257,998281,992110,997265,993076,996594,996395,995998,992721,990976,992793,998468,994374,993194,998331,998214,990292,991945,993715,999373,996245,993866\n", + "Ground Truth: 991470,996760,991257,997506,997019,998536,993381,995168,995257,998281,992110,997265,993076,996594,996395,995998,992721,990976,992793,998468,994374,993194,998331,998214,990292,991945,993715,999373,996245,993936\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 846\n", + "Returned IDs: 990624,997251,992483,992992,999529,996042,995464,992844,999845,996687,997344,999244,998838,991334,997234,991161,993857,995550,997918,994928,997601,992487,990686,993948,993313,990487,995763,995537,992639,993335\n", + "Ground Truth: 990624,997251,992483,992992,999529,996042,995464,992844,999845,996687,997344,999244,998838,991334,997234,991161,993857,995550,997918,994928,997601,992487,990686,993948,993313,990487,995763,995537,996220,992639\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 847\n", + "Returned IDs: 999682,991374,992740,993530,992493,997232,990733,995807,995105,997574,992677,997675,997515,993263,996579,997634,995521,994823,995625,998806,999961,998080,996695,997149,990243,994774,991061,991052,993149,994677\n", + "Ground Truth: 999682,991374,992740,993530,992493,997232,990733,995807,995105,997574,992677,997675,997515,993263,996579,997634,995521,994823,995625,998806,999961,998080,996695,997149,990243,999327,996094,994774,991061,991052\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 848\n", + "Returned IDs: 998960,992511,999374,991444,996959,998211,990469,992147,998644,999263,992416,996218,997704,991362,998500,993250,992854,997192,998921,992546,994850,997543,997301,993457,999968,993364,994673,994939,990515,990215\n", + "Ground Truth: 998960,992511,999374,991444,996959,998211,990469,992147,998644,999263,992416,996218,997704,991362,998500,993250,992854,997192,998921,992546,994850,990760,997543,993457,997301,999968,993364,994673,994939,990515\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 849\n", + "Returned IDs: 990266,991720,993656,992161,992147,997476,991377,990483,992169,994761,994922,992994,997429,996494,991464,990328,999343,997852,992226,996721,990441,998500,999122,999374,992979,990957,990080,992068,995332,996609\n", + "Ground Truth: 990266,991720,993656,992161,992147,997476,991377,990483,992169,994761,997053,994922,992994,997429,996494,991464,990328,999343,997852,992226,996721,990441,998500,995210,999122,999374,992979,990957,990080,992068\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 854\n", + "Returned IDs: 997383,999915,996001,991175,998097,992395,995873,996685,995476,993053,994742,992792,995073,997320,990930,999099,992568,996809,999560,998779,999399,997814,992318,991007,996043,995216,999453,998725,991245,996328\n", + "Ground Truth: 997383,999915,996001,991175,998097,992395,995873,996685,995476,993053,994742,992792,995073,997320,990930,999099,992568,996809,999560,998779,999399,997814,992318,991007,996043,995216,999453,998725,991245,993342\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 855\n", + "Returned IDs: 994080,994760,992558,991977,992170,994119,992718,990776,990487,994254,996633,996809,995179,992052,998686,998480,995503,994956,995159,998243,995491,990908,994334,998331,990345,998372,996328,996391,993839,998370\n", + "Ground Truth: 994080,994760,992558,991977,992170,994119,992718,990776,990487,994254,996633,996809,995179,992052,998686,998480,995503,994956,995159,998243,995491,990908,994334,998331,990345,998372,999179,996328,996391,993839\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 856\n", + "Returned IDs: 993289,992933,993845,993637,990438,993301,999338,992825,993719,994081,998021,995401,990855,999698,992176,993715,998726,998536,996918,995080,997002,990289,997918,990096,998950,999533,999724,996328,999791,999516\n", + "Ground Truth: 993289,992933,993845,993637,990438,993301,999338,992825,993719,994081,998021,995401,990855,999698,992176,993715,998726,998536,996918,995080,997002,990289,997918,990096,998950,991758,999533,999724,996328,999791\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 857\n", + "Returned IDs: 990275,997439,996053,998713,995260,996554,994618,991018,990231,992363,990650,997371,990366,997451,992228,991479,997432,999243,995755,996219,995210,998744,992931,997913,992453,993387,999938,992087,994266,992942\n", + "Ground Truth: 990275,997439,996053,998713,995260,996554,994618,991018,990231,992363,990650,997371,990366,997451,992228,991479,997432,999243,995755,993048,996219,995210,998744,992931,997913,992453,993387,997508,999938,992087\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 860\n", + "Returned IDs: 990275,996336,993853,991260,992384,992452,999647,995014,997976,994776,991829,992969,998899,994992,994192,990724,990862,993168,998514,990925,996727,997571,996714,990356,990957,992994,999089,995034,992911,995279\n", + "Ground Truth: 990275,996336,993853,991260,992384,992452,999647,995014,997976,994776,991829,992969,998899,994992,994192,990724,991893,990862,990893,993168,998514,990925,997022,996727,997571,992909,996714,990356,990957,992994\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 861\n", + "Returned IDs: 998302,994382,997937,991790,995204,991923,994707,990560,995672,997320,993412,997301,995702,992862,995981,991995,998294,994964,995949,997913,998779,990252,998807,996391,998495,995895,995755,990096,995976,997822\n", + "Ground Truth: 998302,994382,999755,997937,991790,995204,991923,994707,990560,995672,997320,993412,997301,995702,992862,995981,991995,998294,994964,995949,997913,998779,990252,998807,996391,998495,995895,997969,995755,990096\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 862\n", + "Returned IDs: 996219,997913,990843,997301,991085,994266,993662,998495,995942,990958,999533,998380,999542,999374,998540,995927,990776,997420,995802,995949,997320,998921,994964,991650,999331,994850,997855,995260,996318,996811\n", + "Ground Truth: 996219,997913,990843,997301,991085,994266,993049,993662,998495,992946,995942,990958,999533,998380,997741,999542,999374,998540,995927,990776,997420,995802,995949,997320,998921,994964,991650,999331,994850,997855\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 863\n", + "Returned IDs: 999912,992337,998115,994972,990366,997453,999544,999876,999949,996052,990179,995107,998122,990585,995222,990089,992047,999791,992907,995716,993139,999711,990043,995818,998349,996001,999898,997301,997796,997028\n", + "Ground Truth: 999912,992337,998115,994972,990366,997453,999544,999876,999949,996052,990179,995107,998122,990585,995222,990089,992047,999791,996947,992907,995716,993139,999711,990043,995818,998349,996001,999898,997301,997796\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 866\n", + "Returned IDs: 998558,994679,994098,995691,996882,999435,998993,990796,998652,994193,990765,991541,999171,994605,992056,994080,995618,990852,993718,996428,993267,994689,997864,990487,996413,990133,999158,998855,996930,992435\n", + "Ground Truth: 998558,994679,994098,995691,996882,999435,998993,990796,998652,994193,990765,991541,999171,994605,992056,994080,995618,990852,993718,996428,993267,994689,997864,990487,996413,990133,999158,998855,996930,990189\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 867\n", + "Returned IDs: 999805,996867,998162,994409,997844,998943,993767,997868,990361,994730,990631,995702,994060,995766,996786,992582,997031,997723,995052,994954,993209,992063,995355,998181,993649,999459,991603,991508,998962,993856\n", + "Ground Truth: 999805,996867,998162,994409,997844,998943,993767,993466,997868,990361,994730,990631,995702,994060,995766,996633,996786,992582,997031,997723,995052,994954,993209,992063,995355,998181,993649,999459,991603,991508\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 869\n", + "Returned IDs: 996411,995482,997374,993466,995020,995002,990435,999627,998156,990293,999599,993209,999785,994730,992024,998221,991480,994802,999179,995274,995633,998067,991075,995401,990758,998913,995374,995289,994817,993747\n", + "Ground Truth: 996411,995482,997374,993466,995020,995002,990435,999627,998156,990293,999599,994142,993209,999785,994730,991439,992024,998221,991480,994802,998586,999179,995274,995633,998067,991075,995401,990758,998913,995374\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 872\n", + "Returned IDs: 996362,995783,994313,996087,992997,993497,997746,995184,990872,990149,995464,991285,994280,990343,997244,990021,994196,999893,996911,991111,999759,996276,990103,997713,993256,993478,993621,991981,992784,994541\n", + "Ground Truth: 996362,995783,994313,996087,992997,993497,997746,995184,990872,990149,997997,995464,991285,994280,990343,997244,990021,994196,999893,996911,991111,999759,996276,990103,997713,993256,993478,993621,991981,992784\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 873\n", + "Returned IDs: 998382,996371,993561,995289,997386,995401,995200,990137,996219,991179,996042,992068,993433,999367,995535,994307,994577,996769,997406,991650,991996,998045,998495,999598,995923,991921,996805,990435,990096,990628\n", + "Ground Truth: 998382,996371,993561,999068,995289,997386,995401,995200,990137,996219,991179,996042,992068,993433,995535,999367,994307,994577,996769,997406,991650,991996,998045,998495,999598,995923,991921,996805,990435,990096\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 874\n", + "Returned IDs: 991255,997914,997542,993738,997918,991948,990814,998697,995281,992979,994552,992012,998427,994280,993143,996328,994163,993256,996305,993079,997251,997806,997555,990776,998860,990634,998123,993976,995401,999820\n", + "Ground Truth: 991255,997914,997542,993738,997918,991948,990814,998697,995281,998567,992979,994552,992012,998097,991447,998427,994280,994972,993143,996328,994163,993256,996305,993079,995600,997251,997806,997555,990776,998860\n", + "Recall: 0.8333\n", + "\n", + "Query ID: 875\n", + "Returned IDs: 995922,998586,998790,994537,996395,991624,993381,991610,996424,990429,992136,997009,990912,991667,990167,991571,993995,990194,999216,996209,992721,990450,994643,996809,993194,991513,991332,991975,990991,994451\n", + "Ground Truth: 995922,998586,998790,994537,996395,991624,993381,991610,996424,990429,992136,997009,990912,991667,990167,991571,993995,990194,999216,994981,996209,992721,990450,994643,996809,993194,991513,991332,991975,990991\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 877\n", + "Returned IDs: 990269,997320,994964,992331,992744,999159,995357,994873,998470,994225,995274,993168,998764,999437,995336,991433,993066,998228,999340,995438,993909,998365,991776,990210,999740,997178,994584,990535,996219,996401\n", + "Ground Truth: 990269,997320,994964,997920,992331,992744,999159,995357,994873,998470,994225,995274,993168,998764,999437,995336,991433,993066,998228,999340,995438,993909,998365,991776,990210,997178,999740,994584,990535,996219\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 878\n", + "Returned IDs: 995955,998242,997215,990125,994305,995503,992306,992824,991333,995804,990538,992170,990721,990192,991034,993140,991834,998260,999908,992490,992113,997849,997547,999430,999639,996742,996510,995724,991977,997055\n", + "Ground Truth: 995955,998242,997215,990125,994305,995503,992306,992824,991333,995804,990538,992170,990721,990192,991034,993140,996629,991834,998260,999908,992490,992113,997849,997547,999430,999639,996742,994155,996510,995724\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 879\n", + "Returned IDs: 997529,993725,994730,998710,994742,991185,995633,994830,999377,992424,993166,990887,998106,993561,993053,995289,998913,999740,990681,994409,996411,993920,996328,992260,993209,994013,994419,996867,992230,996132\n", + "Ground Truth: 997529,993725,994730,998710,994742,991185,995633,994830,999377,992424,993166,990887,998106,993561,993053,995289,998913,999740,990681,997167,994409,996411,993920,996328,992260,993209,994013,994419,996867,992230\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 880\n", + "Returned IDs: 992411,995552,992558,993920,995309,991851,998652,999206,996953,990843,994418,993474,996878,993077,995618,999305,991337,990279,996140,990345,998950,999540,993742,991263,993456,999908,990989,993534,990308,992318\n", + "Ground Truth: 992411,995552,992558,993920,995309,991851,998652,999206,996953,990843,994418,993474,996878,993077,995618,999305,991337,990279,996140,990345,994817,998950,999540,992966,993742,991263,990657,993456,999908,990989\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 882\n", + "Returned IDs: 992032,992293,991076,991203,996225,991954,996028,990662,993561,990811,993725,991087,998476,993857,994835,998823,991603,997529,991958,999286,990399,990097,990033,992636,994677,998197,991301,999922,998593,997554\n", + "Ground Truth: 992032,992293,991076,991203,996225,991954,996028,990662,993561,990811,993725,991087,998476,993857,994835,998823,991603,997529,991958,999286,990399,990097,990033,992636,994677,991771,998197,991301,991466,999922\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 883\n", + "Returned IDs: 997362,990958,996504,993813,997192,990925,996316,999789,998692,995755,992204,999483,998294,990118,997984,997371,995090,997320,999327,993518,991958,995132,995985,995367,998649,997911,997889,992789,995884,997620\n", + "Ground Truth: 997362,990958,996504,993813,997192,990925,996316,999789,998692,995755,992204,999483,998294,990118,997984,997371,995090,997320,999327,993518,991958,995132,995985,991009,995367,998649,997911,997889,993902,993982\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 887\n", + "Returned IDs: 995156,991432,995955,995562,999639,997032,997775,992435,996065,998260,990721,991157,994305,998142,994163,996592,997739,992497,995618,995983,991801,996510,992824,997614,990413,996884,990538,997420,991817,992095\n", + "Ground Truth: 995156,991432,995955,995562,999639,997032,997775,992435,996065,998260,990721,991157,994305,998142,996511,990210,994163,996592,997739,992497,995618,992350,995983,996283,991801,996510,992824,997614,990413,996884\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 889\n", + "Returned IDs: 995159,990355,998507,994929,996211,994541,996413,997746,998755,997722,996344,996276,992489,990931,994094,997879,997290,992445,994336,992313,994913,996430,997814,998174,992052,997420,992832,999179,992981,992934\n", + "Ground Truth: 995159,990355,998507,994929,996211,994541,996413,997746,998755,997722,996344,996276,992489,990931,994094,997879,997290,992445,994336,992313,999974,994913,996430,993621,997814,998174,992052,997420,992832,999179\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 891\n", + "Returned IDs: 990957,993283,992668,995752,991677,999561,996894,994852,999806,995974,998600,995580,998455,990949,995562,999714,995443,991801,992309,993781,998495,999898,993222,994979,995519,997903,991923,992124,992423,997864\n", + "Ground Truth: 990957,993283,992668,995752,991677,999561,996894,994852,999806,995974,998600,995580,998232,998455,990949,995562,999714,995443,991801,992309,993781,998495,999898,993222,997032,994979,995519,997903,991923,992124\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 894\n", + "Returned IDs: 994336,997309,993877,994280,992933,995861,996328,991378,994119,994956,999338,991334,991720,998076,992147,990998,993335,998123,997822,997159,998950,996855,995438,999533,992789,995125,998369,993929,996050,991007\n", + "Ground Truth: 994336,997309,993877,994280,993775,992933,995861,996328,991378,994119,994956,999338,991334,991720,998076,992147,990998,993335,998123,997822,997159,998950,996855,995438,999533,992789,995125,998369,993929,996050\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 897\n", + "Returned IDs: 997475,997652,991153,991075,997480,994891,993475,993621,993611,990696,999496,990600,992480,994765,993055,994558,995600,994313,997714,999205,998405,993165,992449,996856,990776,996429,994051,994307,994280,991723\n", + "Ground Truth: 997475,997652,991153,991075,997480,994891,993475,993621,993611,990696,999496,990600,992480,994765,991426,993055,995145,994558,995600,994313,997714,999205,998405,993165,992449,996856,990776,996429,994051,994307\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 898\n", + "Returned IDs: 991593,995260,999374,991999,994922,990278,998734,997620,998899,996351,990545,994497,991352,991889,999676,997985,990958,991172,999810,993885,992203,994776,990760,990026,991406,991720,994266,999658,995443,993387\n", + "Ground Truth: 991593,995260,992432,999374,991999,994922,990278,998734,997620,998899,996351,990545,994497,991352,991889,999676,997985,990958,991172,999810,993885,992203,997591,994776,990760,990026,991406,991720,994266,999658\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 899\n", + "Returned IDs: 997992,993504,995918,991595,997790,999735,996067,999811,992082,993609,996082,990494,992747,993737,995167,993024,991212,992665,994134,993071,990434,992864,991741,996196,996599,994651,999093,991425,998732,991596\n", + "Ground Truth: 997992,993504,995918,991595,997790,999735,996067,999811,996986,992082,993609,996082,990494,992747,993737,995167,993024,991212,992665,994134,993071,990434,992864,993611,991741,994815,996196,996599,994651,999093\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 901\n", + "Returned IDs: 995986,992718,992258,994163,994072,997689,998694,999908,999359,996163,991637,996926,997395,995301,994165,994195,993463,992645,996669,992558,997864,998079,997692,996214,991430,999432,999385,993609,995436,990367\n", + "Ground Truth: 995986,992718,992258,994163,994072,997689,998694,999908,999359,996163,991637,996926,997395,995301,994165,994195,993463,992645,991172,998758,996669,992558,997864,998079,997692,996214,991430,999432,999385,993609\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 903\n", + "Returned IDs: 999426,991984,990142,999769,993257,991013,998753,994971,997551,993785,992328,994489,998498,994789,996075,993582,991218,992635,995515,998734,990648,997584,995033,996631,995108,993675,996829,996840,998227,999132\n", + "Ground Truth: 999426,991984,990142,999769,993257,991013,998753,994971,997551,993785,992328,994489,998498,994789,996075,993582,991218,992635,995515,998734,990648,997584,995033,997255,996631,995108,993675,996829,996840,998227\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 904\n", + "Returned IDs: 992827,998779,994382,993744,999788,995300,992748,995200,994595,990722,998543,997529,994682,991334,998765,990934,996219,993856,991590,995401,994742,991885,995374,995861,990814,997933,994783,990701,998557,998862\n", + "Ground Truth: 992827,998779,994382,993744,999788,995300,992748,999916,995200,994595,990722,998543,997529,994682,991334,998765,997348,990934,996219,993856,993412,991590,995401,994742,991885,995374,995861,990814,997933,991986\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 907\n", + "Returned IDs: 995802,994778,999710,993492,992442,997006,992443,999982,993504,997483,993852,995826,992309,997913,992119,994670,992931,992228,994831,990231,995161,997605,990957,996993,992066,990890,996785,997320,998170,991873\n", + "Ground Truth: 995802,994778,999710,993492,992442,997006,991009,992443,999982,993504,997483,993852,993048,995826,992309,997913,992119,994670,992931,992228,994831,990231,995161,997605,990957,996993,997579,992066,990890,996785\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 912\n", + "Returned IDs: 999079,990096,999831,996918,999771,997906,998849,997305,999367,994955,992052,992944,996001,995203,994718,994226,995932,995536,998495,996124,995332,994081,997320,994643,996915,995401,994661,991498,994901,993609\n", + "Ground Truth: 999079,990096,999831,996918,999771,997906,998849,997305,999367,994955,992052,992944,996001,995203,995737,994718,994226,995932,995536,998495,996124,995332,994081,997320,994643,996915,995401,994661,991498,994901\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 913\n", + "Returned IDs: 996868,996001,993174,998934,992072,995701,997920,998593,995216,992610,994419,994610,995823,998186,998710,996147,998349,995855,996401,991000,997897,991958,999590,993738,994280,996075,997687,993559,998382,993280\n", + "Ground Truth: 996868,996001,993174,998934,992072,995701,997920,998593,995216,992610,994419,994610,995823,998186,998710,996147,998349,995855,996401,991000,997897,991958,999590,993738,994280,996075,997687,993559,998382,990557\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 916\n", + "Returned IDs: 994673,992833,994160,997192,990724,991782,997826,993648,993412,998458,991585,999328,996111,991361,990760,992284,991743,995702,996721,992994,999263,996532,991602,997880,999374,992147,991213,990432,991958,998899\n", + "Ground Truth: 994673,992833,994160,991900,997192,990724,991782,997826,993648,993412,998458,991585,999328,996111,991361,990760,992284,991743,991534,990462,995702,996721,992994,999263,996532,991602,997880,999374,992147,998215\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 917\n", + "Returned IDs: 994013,994203,995544,992098,996898,998060,993377,991057,999689,997059,998162,994412,991507,999023,995216,993217,996328,992000,994409,994488,996001,999402,998045,997554,991255,996765,995766,997466,997566,997020\n", + "Ground Truth: 994013,994203,995544,992098,996898,998060,993377,991057,999689,997059,998162,994412,991507,999023,995216,993217,996328,992000,994409,994488,996001,999402,998045,997554,991255,996765,995766,997466,996322,997566\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 918\n", + "Returned IDs: 991075,999472,990315,992156,997798,995455,996275,995633,995613,998017,996411,995600,994783,994879,995289,994431,995131,999064,999500,994558,998105,998982,999949,997079,992449,991315,993725,992079,995002,995831\n", + "Ground Truth: 991075,999472,990315,992156,997798,996275,995455,995633,995613,998017,996411,995600,994783,994879,995289,994431,995131,999064,999500,994558,998105,998982,994130,999949,997079,992449,991315,993725,992079,995002\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 919\n", + "Returned IDs: 994342,992780,998631,994577,998109,996475,995776,992068,992070,990271,998568,994984,999806,997331,999780,996219,993055,997990,994558,999430,998495,997909,995895,997554,997480,992628,992423,992655,997529,993053\n", + "Ground Truth: 994342,992780,998631,994577,998109,996475,995776,992068,992070,990271,998568,994984,999806,997331,999780,996219,993055,997990,994558,999430,998495,997909,995895,997116,997554,997480,992628,992423,993678,992655\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 922\n", + "Returned IDs: 996840,996686,995495,998582,998523,995515,993028,997486,991392,993091,998244,999524,994195,990026,998692,995004,991485,999453,993143,991462,990300,992309,999613,998996,996351,992328,990702,998837,991149,995588\n", + "Ground Truth: 996840,996686,995495,998582,998523,995515,993028,997486,991392,993091,998244,999524,994195,990026,998692,995004,991485,999453,993143,991462,990300,992309,999613,998996,996351,995401,992328,990702,998837,991149\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 925\n", + "Returned IDs: 992449,990220,993856,994998,993437,997975,997103,993939,990531,990887,997544,998067,997529,998710,998733,999532,997459,999054,995778,994830,999496,994185,999279,998910,995083,993100,990435,991769,999118,995873\n", + "Ground Truth: 992449,990220,993856,994998,993437,997975,997103,993939,990531,990887,997544,998067,997529,998710,998733,999532,997459,999054,995778,994830,999496,994185,999279,998910,995083,993100,990435,991769,999118,997554\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 926\n", + "Returned IDs: 994830,996132,994204,990340,990711,994742,994783,995633,998913,990012,998710,997319,993053,998196,996411,991330,999377,995289,994529,992289,996302,993433,996049,997529,995948,995200,996809,990435,993166,997797\n", + "Ground Truth: 994830,996132,994204,990340,990711,994742,994783,995633,998913,990012,998710,997319,993053,998196,996411,991330,999377,995289,994529,992289,996302,993433,996049,995401,997529,995948,995200,996809,990435,993166\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 931\n", + "Returned IDs: 995299,990074,998803,993413,992456,996183,998949,997340,996986,990730,996408,992372,998903,993335,994682,990998,998716,990700,991919,997577,995393,996357,992628,995313,990173,992664,994269,993917,993101,991382\n", + "Ground Truth: 995299,990074,998803,993413,992456,996183,998949,997340,996986,990730,996408,992372,998903,993335,994682,990998,998716,990700,991919,997577,995393,996357,992628,995313,990173,992664,994269,993917,997116,993101\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 932\n", + "Returned IDs: 997918,993289,997002,995826,993504,993492,993512,999900,996511,994449,992571,993781,992029,994959,999840,997790,994268,990096,998090,996062,999019,990399,997320,997977,990930,999355,994280,997586,991206,996592\n", + "Ground Truth: 997918,993289,997002,995826,993504,993492,993512,999900,996511,994449,992571,993781,992029,994959,999840,997790,994268,990096,998090,996062,999019,990399,997320,997977,990930,999355,991648,994280,997586,991206\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 933\n", + "Returned IDs: 999784,996001,996028,998302,997371,997002,997320,998921,999806,994266,995560,993820,995596,998720,997913,993834,994964,993342,996840,993902,997301,990532,993906,996096,995886,993260,994689,995852,992330,997506\n", + "Ground Truth: 999784,996001,996028,998302,997371,997002,997320,998921,999806,994266,995560,993820,995596,998720,997913,993834,994964,993342,996840,993902,997301,990532,995375,993906,996096,995886,993260,994689,995852,992330\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 935\n", + "Returned IDs: 995981,996609,990444,996648,991721,994069,999374,991593,993412,990990,996085,998302,998777,997320,998246,996805,997301,995266,998339,993186,998849,991422,990585,998974,991602,993049,990096,990146,992291,991809\n", + "Ground Truth: 995981,996609,990444,996648,991721,994069,999374,991593,993412,990990,996085,998302,998777,999046,997320,998246,996805,997301,995266,998339,993186,998849,991422,990585,998974,991602,993049,990096,990146,992291\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 936\n", + "Returned IDs: 997700,999581,999054,991507,990537,991722,995106,994861,995268,993209,992448,993313,995778,996607,990372,997738,995820,998162,998910,998283,999279,995518,996505,994380,990924,991424,996813,995401,991699,995521\n", + "Ground Truth: 997700,999581,999054,991507,990537,991722,995106,994861,995268,993209,992448,993313,995778,996607,990372,997738,995820,998162,998910,998283,999279,995518,996505,994380,990924,991424,993046,996813,995401,991699\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 938\n", + "Returned IDs: 998777,996219,999676,995401,999453,998530,998797,992309,998495,993355,994318,990843,996840,998302,992950,990974,992328,994850,995016,991585,994195,993412,994964,990432,995899,992216,995374,999302,997554,996033\n", + "Ground Truth: 998777,996219,999676,995401,999453,998530,998797,992309,998495,993355,994318,990843,996840,998302,992950,990974,992328,994850,995016,991585,994195,993431,993412,994964,995899,990432,992216,995374,999302,997554\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 940\n", + "Returned IDs: 994897,993342,999806,993617,997335,999430,992628,991271,993528,992807,995393,990173,990194,991846,999555,994155,998787,997126,994150,996809,999834,996660,995895,997196,994122,993076,994269,996184,996944,994834\n", + "Ground Truth: 994897,993342,999806,993617,997335,999430,992628,995039,991271,993528,992807,995393,990173,990194,991846,999555,994155,998214,998787,997126,994150,996809,999834,996660,995895,997196,994122,993076,994269,996184\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 942\n", + "Returned IDs: 998793,999476,997215,992802,997744,990782,990150,995552,990755,994965,995724,990192,990908,992608,999639,990334,999164,998242,996196,990930,997548,991791,998495,990538,994661,999908,995979,995430,995372,999294\n", + "Ground Truth: 998793,999476,997215,992802,997744,990782,990150,995552,990755,994965,994536,995724,990192,990908,992608,999639,990334,999164,998242,996196,990930,997548,991791,998495,990538,994661,999908,995979,995430,995372\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 943\n", + "Returned IDs: 994879,998347,992582,997728,996673,992760,994265,997353,998225,992502,991431,992293,996948,992862,997304,995981,999761,992481,990862,998708,994304,998986,994894,998697,993072,999243,993809,997620,995927,992183\n", + "Ground Truth: 994879,998347,993222,992582,997728,996673,992760,994265,997353,998225,992502,991431,992293,996948,992862,997304,991680,995981,999761,992481,990862,998708,994304,998986,994894,998697,993072,999243,993809,997620\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 945\n", + "Returned IDs: 993209,993299,996867,994317,990638,993586,997544,990082,994742,997723,997319,992582,990126,997077,993549,995030,994587,991943,995211,996302,998913,995633,991664,990493,998910,992289,995268,993920,996411,991639\n", + "Ground Truth: 993209,993299,996867,994317,990638,993586,997544,990082,994742,997723,997319,992582,990126,997077,993549,995078,995030,994587,991943,995211,996302,992667,998913,995633,996277,991664,990493,998910,992289,995268\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 946\n", + "Returned IDs: 995547,999530,994409,999367,995994,998545,991061,997723,993717,993561,991923,997191,996042,994948,996769,994026,996038,996918,997406,995401,995507,997832,991459,994002,997675,993143,997289,991052,995923,993271\n", + "Ground Truth: 995547,999530,994409,999367,995994,998545,991061,997723,993717,993561,991923,997191,996042,994948,996769,994026,996038,996918,997406,995401,995507,997832,991459,993122,994002,993143,997675,997289,998250,991052\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 948\n", + "Returned IDs: 996001,994449,995536,990532,996661,997586,997320,996824,992862,994200,999444,996178,999639,991630,992635,999587,995672,990814,993504,992469,999974,995983,995250,993157,990399,991981,998827,999453,995401,994334\n", + "Ground Truth: 996001,994449,995536,990532,996661,997586,997320,996824,996511,992862,996368,994200,999444,996178,999639,991630,992521,992635,999587,995672,990814,993504,992469,999974,994343,995983,995250,993157,990399,991981\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 951\n", + "Returned IDs: 999076,993355,997539,994922,990300,998424,990809,990580,995210,995515,997747,999582,993688,998236,998751,993237,998837,999469,994789,992612,991609,991392,997291,996561,992979,999097,998401,997772,997818,992950\n", + "Ground Truth: 999076,993355,997539,994922,990300,998424,990809,990580,995210,995515,997747,999582,995430,993688,998236,998751,993237,998837,999469,994789,992612,993222,991609,991392,997291,996561,992979,999097,996793,998401\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 954\n", + "Returned IDs: 995840,997807,998414,994782,998668,997586,998496,997004,995927,999210,999941,998068,993114,995944,999931,993689,993878,997364,990866,999898,995971,993624,997305,991259,998814,998708,990373,993831,999218,997552\n", + "Ground Truth: 995840,997807,998414,994782,998668,997586,998496,997004,994756,995927,999210,999941,998068,993114,995944,999931,993689,993878,997364,990866,999898,998861,991006,995971,993624,997305,991259,998814,998708,990373\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 956\n", + "Returned IDs: 992630,999343,991335,992384,996219,990934,990171,996401,994100,990974,990448,996143,999331,998779,997571,998170,997371,997920,996790,999102,998550,997006,990275,999647,996777,991862,990315,996727,998500,990666\n", + "Ground Truth: 992630,999343,991335,992384,996219,990934,990171,992931,996401,994100,990974,990448,996143,999331,998779,997571,998170,997371,997920,996790,999102,998550,997006,990275,999647,996777,991862,990315,996727,998500\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 960\n", + "Returned IDs: 998335,990627,992862,991304,996456,996104,995991,996594,992270,994564,990802,991447,995231,994742,995168,990998,996408,999319,993076,999334,995780,996611,998279,992636,996728,991098,994498,997506,995626,993638\n", + "Ground Truth: 998335,990627,992862,991304,996456,996104,994796,995991,996594,992270,994564,990802,991447,995231,994742,995168,990998,996408,999319,993076,999334,995780,996611,998279,992636,996728,991098,994498,997506,995626\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 962\n", + "Returned IDs: 995336,997301,995438,994964,995961,994757,998246,999144,994360,999331,999710,997320,990949,995942,993492,992384,997470,990231,998302,994100,998090,990958,996592,995886,993241,992228,994081,990957,990096,991883\n", + "Ground Truth: 995336,997301,995438,994964,995961,994757,998246,999144,994360,999331,993154,999710,997320,990949,995942,993492,992384,997470,990231,998302,997775,999301,994100,998090,990958,996592,995886,993241,992228,994081\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 963\n", + "Returned IDs: 992135,991459,998623,990537,998720,990682,996001,995968,990457,996060,990502,991154,996247,990005,993673,996812,995027,994809,997091,994380,999453,999543,996417,990616,997965,996759,999724,995932,998068,995937\n", + "Ground Truth: 992135,991459,998623,990537,998720,990682,996001,995968,990457,997140,996060,990502,991154,996247,990005,993673,996812,995027,993131,994809,997091,994380,999453,992124,999543,996417,990616,997965,996759,998029\n", + "Recall: 0.8667\n", + "\n", + "Query ID: 965\n", + "Returned IDs: 994964,997379,995791,993035,992944,994777,996397,999210,998950,996811,993696,993633,993215,992612,990310,998665,995365,994922,996915,991999,999676,990690,993355,992169,993809,996093,994318,993561,992334,995596\n", + "Ground Truth: 994964,997379,995791,993035,992944,994777,996397,999210,998950,996811,993831,993696,993633,994089,993215,992612,990310,998665,995365,994922,996915,991999,999676,990690,993355,992169,993809,996093,994318,993561\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 966\n", + "Returned IDs: 998797,999995,997040,990146,999262,995884,997320,990925,994964,998302,992546,996959,990936,990958,991852,995947,997586,992896,996785,997911,993484,998246,997913,996075,995336,997822,999343,999242,999374,994366\n", + "Ground Truth: 998797,999995,997040,990146,999262,995884,997320,990925,994964,998302,995873,992546,996959,990936,990958,991852,995947,997586,992896,996785,997911,993484,998246,997913,996075,995336,997822,999343,996296,999242\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 967\n", + "Returned IDs: 995349,997685,999079,992664,997673,991148,993866,997906,998468,999443,999576,992209,995969,995231,992275,994661,997369,990096,997666,993965,995064,995895,996733,991477,995695,993445,996124,995735,995393,994751\n", + "Ground Truth: 995349,997685,999079,992664,997673,991148,993866,997906,998468,999443,999576,992209,995969,995231,992275,994661,997369,990096,997666,993965,995064,995895,996733,991477,995695,993445,996124,995735,992752,995393\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 969\n", + "Returned IDs: 991110,990729,999737,998022,991767,998318,994063,991887,994088,995373,999662,995449,997131,995480,992604,990256,999445,994122,992133,996236,990577,996730,994577,998629,993137,997942,995080,990345,994539,993185\n", + "Ground Truth: 991110,990729,999737,998022,991767,998318,994063,991887,994088,995373,999662,994956,995449,997131,995480,992604,990256,999445,994122,992133,996236,990577,996730,994577,998629,993137,997942,995080,990345,994539\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 970\n", + "Returned IDs: 993484,994964,993648,993642,993238,997320,999374,990995,992047,990399,999453,995755,993389,996959,990936,990224,997737,996785,993143,993885,993412,995961,993663,998294,991743,994922,995886,990958,994673,998692\n", + "Ground Truth: 993484,994964,993648,993642,993238,997320,999374,990995,992047,990399,999453,995755,993389,996959,990936,990224,997737,996785,993143,993885,993412,995961,993663,992334,993400,998294,991743,994922,995886,990958\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 972\n", + "Returned IDs: 996001,993053,994106,992182,999352,995200,996219,993883,995873,994382,997909,990435,998725,999653,995216,994689,990557,991464,990752,995823,995289,993438,996657,994529,994783,998982,991212,999862,995222,996277\n", + "Ground Truth: 996001,993053,994300,999151,994106,992182,999352,995200,998568,996219,993883,995873,994382,997909,990435,995384,998725,999653,995216,994689,996411,997837,990557,991464,992384,995823,990752,995289,995198,993438\n", + "Recall: 0.7333\n", + "\n", + "Query ID: 973\n", + "Returned IDs: 990296,992161,998221,993412,998649,998495,990957,999898,990399,997192,996869,992107,995942,990435,993656,996001,990934,991335,997582,991311,992087,996219,999144,999710,995289,998039,993168,991096,995755,995536\n", + "Ground Truth: 990296,992161,998221,993412,998649,998495,990957,999898,990399,997920,997192,996869,992107,995942,990435,993656,996001,990934,991335,997582,991311,992087,996219,999144,999710,995289,992124,995899,998039,993168\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 976\n", + "Returned IDs: 995958,995778,997023,992473,997723,993209,996376,994453,995401,996429,996226,995701,999675,998067,997601,997204,997216,991507,991044,996232,998495,997732,997544,993948,991747,993313,996328,991958,996506,998045\n", + "Ground Truth: 995958,995778,997023,992473,997723,993209,996376,994453,995401,996429,996226,995701,999675,995536,998067,997601,997204,997216,991507,991044,999977,996232,998495,997732,997544,993948,990814,991747,993313,996328\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 983\n", + "Returned IDs: 995295,996580,995283,994730,990528,992226,991464,993725,990531,990930,993585,990315,996411,994742,994682,993623,990614,990076,990133,995289,994783,999798,990811,992680,996136,993053,997141,993705,997529,997340\n", + "Ground Truth: 995295,996580,995283,994730,990528,992226,991464,993725,990531,990930,993585,990315,996411,994742,994682,993623,990614,990076,990133,995289,994783,999798,990811,992680,996136,993053,997141,991629,993705,997529\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 987\n", + "Returned IDs: 997204,992169,993288,995289,995374,999587,995359,999099,992685,995685,991151,997918,998530,993909,993412,991810,994409,996328,999820,992979,992239,995401,997395,997002,997630,994906,990337,996903,997447,995994\n", + "Ground Truth: 997204,992169,993288,995289,995374,999587,995359,999099,992685,995685,991151,997918,998530,993909,993412,991810,994409,996328,996220,999820,992979,992239,995401,997395,997002,997630,994906,990337,996903,997447\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 989\n", + "Returned IDs: 997267,990129,991544,995891,996472,992448,992801,996042,991958,995521,994409,997884,995864,998045,997909,998703,995200,995198,993433,994002,994742,991671,994954,994413,993127,993561,998623,992123,995702,997077\n", + "Ground Truth: 997267,990129,991544,995891,996472,992448,992801,996042,991958,995521,994409,997884,995864,995873,998045,997909,998703,990681,995200,995198,993433,994002,994742,991671,994954,994413,993127,993561,998623,992123\n", + "Recall: 0.9333\n", + "\n", + "Query ID: 991\n", + "Returned IDs: 996571,991975,996120,993381,997145,997965,994724,991987,993641,991513,991038,999653,997369,995780,996920,997906,991844,997265,993629,992945,998586,994374,998861,990450,995168,996209,990991,995231,992862,996326\n", + "Ground Truth: 996571,991975,996120,993381,997145,997965,994724,991987,993641,991513,991038,999653,997369,995780,996920,997906,991844,997265,993629,992945,998586,994374,998861,990450,995168,992233,996209,990991,995231,992862\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 994\n", + "Returned IDs: 994102,990092,991862,991335,996052,999647,990957,990231,996219,992134,994831,990545,997439,997913,996143,992114,995942,991743,999343,994192,993264,997320,990171,999144,993135,992931,996053,995865,998302,990949\n", + "Ground Truth: 994102,990092,991862,991335,996052,999647,990957,990231,996219,992134,994831,990545,997439,997913,996143,992114,995942,991743,995844,999343,999993,994192,993264,997320,990171,999144,993135,992931,996053,994898\n", + "Recall: 0.9000\n", + "\n", + "Query ID: 995\n", + "Returned IDs: 996627,994751,996411,993070,997729,990011,990012,990528,990743,996906,994663,995295,993725,992230,992680,992408,999740,999922,994830,990392,993240,994730,995076,996011,992024,993705,993906,995435,998030,991464\n", + "Ground Truth: 996627,994751,996411,993070,997729,990011,990012,990528,990743,996906,994663,995295,993725,992230,992680,992408,999740,999922,994830,990392,993240,994730,995076,996011,992024,996147,993705,993906,995435,998030\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 996\n", + "Returned IDs: 995193,992521,998527,992046,992657,993021,995820,995640,997023,996581,999047,999394,991922,991102,992917,994508,990099,992311,999731,998075,993107,994185,994690,998255,990295,992595,997554,998910,993998,997320\n", + "Ground Truth: 995193,992521,998527,992046,992657,993021,995820,995640,997023,996581,999047,999394,991922,991102,992917,994508,990099,992311,999731,998075,993107,994185,994690,998255,999992,990295,992595,997554,998910,993998\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 997\n", + "Returned IDs: 991052,997675,996918,993805,996506,991061,997723,995401,991529,993014,996505,995521,996879,998397,999646,991747,993931,998593,994280,995080,993313,990537,994662,993600,996042,992521,995994,994095,992933,997411\n", + "Ground Truth: 991052,997675,996918,993805,996506,991061,997723,995401,991529,993014,996505,995521,996879,998397,999646,991747,993931,998593,994280,995080,993313,990537,994662,993600,998021,996042,992521,995994,994095,992933\n", + "Recall: 0.9667\n", + "\n", + "Query ID: 999\n", + "Returned IDs: 999162,992654,991977,999638,995560,998334,999387,994010,994672,998118,992718,991540,998142,995503,993018,998280,990930,997407,991432,991157,997320,990843,990926,999954,997816,993976,996262,992124,990487,992944\n", + "Ground Truth: 999162,992654,991977,999638,995560,998334,999387,994010,994672,998118,992718,991540,998142,995503,993018,998567,998280,990930,997407,991432,991157,997320,990843,990926,999954,997816,993976,996262,992124,990487\n", + "Recall: 0.9667\n", + "\n", + "Total query IDs with recall < 1.0: 542\n", + "IDs: [np.int64(1), np.int64(2), np.int64(4), np.int64(5), np.int64(6), np.int64(8), np.int64(10), np.int64(19), np.int64(21), np.int64(22), np.int64(23), np.int64(24), np.int64(26), np.int64(29), np.int64(31), np.int64(34), np.int64(35), np.int64(40), np.int64(41), np.int64(43), np.int64(45), np.int64(46), np.int64(48), np.int64(52), np.int64(53), np.int64(54), np.int64(56), np.int64(58), np.int64(59), np.int64(60), np.int64(61), np.int64(62), np.int64(63), np.int64(65), np.int64(66), np.int64(67), np.int64(69), np.int64(71), np.int64(74), np.int64(75), np.int64(79), np.int64(81), np.int64(84), np.int64(85), np.int64(88), np.int64(89), np.int64(91), np.int64(93), np.int64(100), np.int64(102), np.int64(104), np.int64(105), np.int64(107), np.int64(108), np.int64(109), np.int64(110), np.int64(112), np.int64(113), np.int64(116), np.int64(117), np.int64(119), np.int64(127), np.int64(128), np.int64(129), np.int64(131), np.int64(132), np.int64(133), np.int64(134), np.int64(135), np.int64(136), np.int64(138), np.int64(139), np.int64(144), np.int64(150), np.int64(151), np.int64(153), np.int64(154), np.int64(156), np.int64(158), np.int64(159), np.int64(160), np.int64(161), np.int64(162), np.int64(163), np.int64(170), np.int64(171), np.int64(172), np.int64(175), np.int64(178), np.int64(179), np.int64(181), np.int64(183), np.int64(184), np.int64(187), np.int64(188), np.int64(191), np.int64(192), np.int64(199), np.int64(200), np.int64(203), np.int64(204), np.int64(208), np.int64(211), np.int64(212), np.int64(213), np.int64(215), np.int64(216), np.int64(218), np.int64(220), np.int64(221), np.int64(223), np.int64(224), np.int64(227), np.int64(228), np.int64(229), np.int64(230), np.int64(231), np.int64(233), np.int64(236), np.int64(240), np.int64(242), np.int64(247), np.int64(249), np.int64(251), np.int64(258), np.int64(259), np.int64(261), np.int64(264), np.int64(266), np.int64(268), np.int64(269), np.int64(270), np.int64(272), np.int64(273), np.int64(275), np.int64(277), np.int64(278), np.int64(281), np.int64(282), np.int64(283), np.int64(284), np.int64(285), np.int64(286), np.int64(287), np.int64(289), np.int64(293), np.int64(294), np.int64(295), np.int64(296), np.int64(297), np.int64(302), np.int64(303), np.int64(304), np.int64(305), np.int64(306), np.int64(308), np.int64(309), np.int64(311), np.int64(312), np.int64(315), np.int64(318), np.int64(319), np.int64(320), np.int64(321), np.int64(322), np.int64(323), np.int64(324), np.int64(325), np.int64(326), np.int64(327), np.int64(328), np.int64(331), np.int64(334), np.int64(337), np.int64(342), np.int64(344), np.int64(345), np.int64(346), np.int64(347), np.int64(349), np.int64(352), np.int64(356), np.int64(357), np.int64(358), np.int64(359), np.int64(360), np.int64(361), np.int64(362), np.int64(366), np.int64(367), np.int64(368), np.int64(370), np.int64(371), np.int64(372), np.int64(376), np.int64(379), np.int64(381), np.int64(382), np.int64(383), np.int64(385), np.int64(389), np.int64(390), np.int64(393), np.int64(394), np.int64(395), np.int64(399), np.int64(402), np.int64(403), np.int64(406), np.int64(407), np.int64(409), np.int64(410), np.int64(411), np.int64(412), np.int64(413), np.int64(417), np.int64(420), np.int64(423), np.int64(424), np.int64(425), np.int64(427), np.int64(429), np.int64(430), np.int64(431), np.int64(432), np.int64(433), np.int64(434), np.int64(437), np.int64(439), np.int64(440), np.int64(441), np.int64(442), np.int64(447), np.int64(448), np.int64(449), np.int64(450), np.int64(451), np.int64(452), np.int64(453), np.int64(455), np.int64(456), np.int64(457), np.int64(458), np.int64(459), np.int64(460), np.int64(461), np.int64(462), np.int64(467), np.int64(469), np.int64(470), np.int64(473), np.int64(475), np.int64(477), np.int64(480), np.int64(481), np.int64(485), np.int64(488), np.int64(491), np.int64(494), np.int64(496), np.int64(498), np.int64(499), np.int64(500), np.int64(501), np.int64(503), np.int64(504), np.int64(505), np.int64(507), np.int64(508), np.int64(509), np.int64(510), np.int64(511), np.int64(512), np.int64(513), np.int64(514), np.int64(515), np.int64(517), np.int64(520), np.int64(521), np.int64(523), np.int64(524), np.int64(527), np.int64(529), np.int64(533), np.int64(534), np.int64(541), np.int64(543), np.int64(544), np.int64(545), np.int64(546), np.int64(548), np.int64(549), np.int64(551), np.int64(552), np.int64(557), np.int64(558), np.int64(561), np.int64(563), np.int64(564), np.int64(565), np.int64(567), np.int64(569), np.int64(571), np.int64(574), np.int64(575), np.int64(576), np.int64(577), np.int64(578), np.int64(581), np.int64(583), np.int64(584), np.int64(586), np.int64(588), np.int64(589), np.int64(591), np.int64(593), np.int64(595), np.int64(596), np.int64(600), np.int64(604), np.int64(605), np.int64(610), np.int64(611), np.int64(612), np.int64(613), np.int64(615), np.int64(616), np.int64(618), np.int64(619), np.int64(620), np.int64(621), np.int64(623), np.int64(625), np.int64(626), np.int64(628), np.int64(629), np.int64(630), np.int64(632), np.int64(633), np.int64(636), np.int64(638), np.int64(639), np.int64(640), np.int64(641), np.int64(643), np.int64(644), np.int64(645), np.int64(646), np.int64(647), np.int64(648), np.int64(649), np.int64(650), np.int64(652), np.int64(654), np.int64(655), np.int64(656), np.int64(658), np.int64(661), np.int64(665), np.int64(666), np.int64(668), np.int64(669), np.int64(670), np.int64(671), np.int64(673), np.int64(674), np.int64(676), np.int64(678), np.int64(680), np.int64(681), np.int64(682), np.int64(688), np.int64(689), np.int64(690), np.int64(691), np.int64(694), np.int64(696), np.int64(697), np.int64(700), np.int64(701), np.int64(704), np.int64(705), np.int64(706), np.int64(708), np.int64(710), np.int64(712), np.int64(713), np.int64(714), np.int64(718), np.int64(719), np.int64(720), np.int64(724), np.int64(726), np.int64(727), np.int64(728), np.int64(730), np.int64(731), np.int64(733), np.int64(734), np.int64(735), np.int64(739), np.int64(740), np.int64(742), np.int64(743), np.int64(744), np.int64(745), np.int64(748), np.int64(749), np.int64(751), np.int64(754), np.int64(758), np.int64(760), np.int64(761), np.int64(762), np.int64(763), np.int64(764), np.int64(765), np.int64(768), np.int64(770), np.int64(771), np.int64(772), np.int64(773), np.int64(774), np.int64(775), np.int64(776), np.int64(777), np.int64(778), np.int64(779), np.int64(781), np.int64(782), np.int64(783), np.int64(787), np.int64(789), np.int64(791), np.int64(792), np.int64(794), np.int64(796), np.int64(800), np.int64(801), np.int64(804), np.int64(805), np.int64(806), np.int64(809), np.int64(810), np.int64(816), np.int64(817), np.int64(818), np.int64(820), np.int64(821), np.int64(822), np.int64(823), np.int64(827), np.int64(828), np.int64(829), np.int64(830), np.int64(832), np.int64(833), np.int64(835), np.int64(839), np.int64(841), np.int64(842), np.int64(844), np.int64(846), np.int64(847), np.int64(848), np.int64(849), np.int64(854), np.int64(855), np.int64(856), np.int64(857), np.int64(860), np.int64(861), np.int64(862), np.int64(863), np.int64(866), np.int64(867), np.int64(869), np.int64(872), np.int64(873), np.int64(874), np.int64(875), np.int64(877), np.int64(878), np.int64(879), np.int64(880), np.int64(882), np.int64(883), np.int64(887), np.int64(889), np.int64(891), np.int64(894), np.int64(897), np.int64(898), np.int64(899), np.int64(901), np.int64(903), np.int64(904), np.int64(907), np.int64(912), np.int64(913), np.int64(916), np.int64(917), np.int64(918), np.int64(919), np.int64(922), np.int64(925), np.int64(926), np.int64(931), np.int64(932), np.int64(933), np.int64(935), np.int64(936), np.int64(938), np.int64(940), np.int64(942), np.int64(943), np.int64(945), np.int64(946), np.int64(948), np.int64(951), np.int64(954), np.int64(956), np.int64(960), np.int64(962), np.int64(963), np.int64(965), np.int64(966), np.int64(967), np.int64(969), np.int64(970), np.int64(972), np.int64(973), np.int64(976), np.int64(983), np.int64(987), np.int64(989), np.int64(991), np.int64(994), np.int64(995), np.int64(996), np.int64(997), np.int64(999)]\n", + "Average Recall across all 1000 queries: 0.9664\n" + ] + } + ], + "source": [ + "#For querying (int filter)- gte filter\n", + "import pyarrow.parquet as pq\n", + "import os\n", + "\n", + "#inputs\n", + "int_rate_query = \"99p\" # change to 1p, 50p, 80p, 99p\n", + "top_k = 30\n", + "\n", + "# Mode 1: Single query id\n", + "# Mode 2: Find all query ids with recall < 1\n", + "mode = 2\n", + "query_id = 457 # only used in mode 1\n", + "\n", + "# Map int_rate_query to filter range\n", + "range_map = {\n", + " \"1p\": [10000, 1000000],\n", + " \"50p\": [500000, 1000000],\n", + " \"80p\": [800000, 1000000],\n", + " \"99p\": [990000, 1000000],\n", + "}\n", + "filter_range = range_map[int_rate_query]\n", + "\n", + "# Build full paths\n", + "dataset_path = os.path.join(base_path, dataset_folder)\n", + "test_parquet_path = os.path.join(dataset_path, \"test.parquet\")\n", + "gt_parquet_path = os.path.join(dataset_path, f\"neighbors_int_{int_rate_query}.parquet\")\n", + "\n", + "# Read parquet files\n", + "test_df = pq.ParquetFile(test_parquet_path).read().to_pandas()\n", + "gt_df = pq.ParquetFile(gt_parquet_path).read().to_pandas()\n", + "\n", + "def run_query(query_id):\n", + " query_row = test_df[test_df[\"id\"] == query_id]\n", + " if query_row.empty:\n", + " print(f\"Query ID {query_id} not found in test.parquet\")\n", + " return None\n", + "\n", + " query_vector = query_row[\"emb\"].values[0]\n", + "\n", + " gt_row = gt_df[gt_df[\"id\"] == query_id]\n", + " if gt_row.empty:\n", + " print(f\"Query ID {query_id} not found in ground truth file\")\n", + " return None\n", + "\n", + " ground_truth = gt_row[\"neighbors_id\"].values[0][:top_k]\n", + "\n", + " results = index.query(\n", + " vector=query_vector,\n", + " top_k=top_k,\n", + " filter=[{\"id\": {\"$gte\": filter_range[0]}}],\n", + " include_vectors=False,\n", + " )\n", + " returned_ids = [int(r[\"id\"]) for r in results]\n", + "\n", + " intersection = len(set(returned_ids) & set(ground_truth))\n", + " recall = intersection / len(ground_truth)\n", + "\n", + " return returned_ids, ground_truth, recall\n", + "\n", + "\n", + "if mode == 1:\n", + " result = run_query(query_id)\n", + " if result:\n", + " returned_ids, ground_truth, recall = result\n", + " print(f\"Query ID: {query_id}\")\n", + " print(f\"Returned IDs: {','.join(map(str, returned_ids))}\")\n", + " print(f\"Ground Truth: {','.join(map(str, ground_truth))}\")\n", + " print(f\"Recall: {recall:.4f}\")\n", + "\n", + "elif mode == 2:\n", + " print(f\"Finding all query IDs with recall < 1.0 ...\\n\")\n", + " low_recall_ids = []\n", + " all_recalls = []\n", + "\n", + " for qid in test_df[\"id\"].values:\n", + " result = run_query(qid)\n", + " if result:\n", + " returned_ids, ground_truth, recall = result\n", + " all_recalls.append(recall)\n", + " if recall < 1.0:\n", + " low_recall_ids.append(qid)\n", + " print(f\"Query ID: {qid}\")\n", + " print(f\"Returned IDs: {','.join(map(str, returned_ids))}\")\n", + " print(f\"Ground Truth: {','.join(map(str, ground_truth))}\")\n", + " print(f\"Recall: {recall:.4f}\\n\")\n", + "\n", + " print(f\"Total query IDs with recall < 1.0: {len(low_recall_ids)}\")\n", + " print(f\"IDs: {low_recall_ids}\")\n", + " print(f\"Average Recall across all {len(all_recalls)} queries: {sum(all_recalls)/len(all_recalls):.4f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'name': 'test_1M_numfilter_latest_master_stability_vaib2',\n", + " 'space_type': 'cosine',\n", + " 'dimension': 768,\n", + " 'sparse_dim': 0,\n", + " 'is_hybrid': False,\n", + " 'count': 200000,\n", + " 'precision': 'int16',\n", + " 'M': 16}" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "index.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#For deleting (int filter) - range operator\n", + "int_rate_delete = \"1p\" # change to 1p, 50p, 80p, 99p\n", + "\n", + "# Map int_rate to range\n", + "range_map = {\n", + " \"1p\": [0, 9999],\n", + " \"50p\": [0, 499999],\n", + " \"80p\": [0, 799999],\n", + " \"99p\": [0, 989999],\n", + "}\n", + "filter_range = range_map[int_rate_delete]\n", + "\n", + "result = index.delete_with_filter([{\"id\": {\"$range\": filter_range}}])\n", + "print(f\"Deleted vectors with id range: {filter_range}\")\n", + "print(result)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Deleted vectors with id < 800000\n", + "800000 vectors deleted\n" + ] + } + ], + "source": [ + "#For deleting (int filter) - gte\n", + "int_rate_delete = \"80p\" # change to 1p, 50p, 80p, 99p\n", + "\n", + "# Map int_rate to lower bound (gte value)\n", + "range_map = {\n", + " \"1p\": 10000,\n", + " \"50p\": 500000,\n", + " \"60p\": 600000,\n", + " \"70p\": 700000,\n", + " \"80p\": 800000,\n", + " \"99p\": 990000,\n", + "}\n", + "gte_value = range_map[int_rate_delete]\n", + "\n", + "result = index.delete_with_filter([{\"id\": {\"$lt\": gte_value}}])\n", + "print(f\"Deleted vectors with id < {gte_value}\")\n", + "print(result)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'name': 'test_1M_vaib_reupsertcheck',\n", + " 'space_type': 'cosine',\n", + " 'dimension': 768,\n", + " 'sparse_dim': 0,\n", + " 'is_hybrid': False,\n", + " 'count': 300000,\n", + " 'precision': 'int16',\n", + " 'M': 16}" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "index.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Upserted 798 vectors, total so far: 798\n", + "Upserted 812 vectors, total so far: 1610\n", + "Upserted 785 vectors, total so far: 2395\n", + "Upserted 813 vectors, total so far: 3208\n", + "Upserted 789 vectors, total so far: 3997\n", + "Upserted 827 vectors, total so far: 4824\n", + "Upserted 796 vectors, total so far: 5620\n", + "Upserted 804 vectors, total so far: 6424\n", + "Upserted 793 vectors, total so far: 7217\n", + "Upserted 787 vectors, total so far: 8004\n", + "Upserted 794 vectors, total so far: 8798\n", + "Upserted 797 vectors, total so far: 9595\n", + "Upserted 797 vectors, total so far: 10392\n", + "Upserted 817 vectors, total so far: 11209\n", + "Upserted 787 vectors, total so far: 11996\n", + "Upserted 802 vectors, total so far: 12798\n", + "Upserted 800 vectors, total so far: 13598\n", + "Upserted 786 vectors, total so far: 14384\n", + "Upserted 811 vectors, total so far: 15195\n", + "Upserted 772 vectors, total so far: 15967\n", + "Upserted 816 vectors, total so far: 16783\n", + "Upserted 796 vectors, total so far: 17579\n", + "Upserted 811 vectors, total so far: 18390\n", + "Upserted 767 vectors, total so far: 19157\n", + "Upserted 816 vectors, total so far: 19973\n", + "Upserted 800 vectors, total so far: 20773\n", + "Upserted 792 vectors, total so far: 21565\n", + "Upserted 800 vectors, total so far: 22365\n", + "Upserted 806 vectors, total so far: 23171\n", + "Upserted 792 vectors, total so far: 23963\n", + "Upserted 799 vectors, total so far: 24762\n", + "Upserted 798 vectors, total so far: 25560\n", + "Upserted 817 vectors, total so far: 26377\n", + "Upserted 813 vectors, total so far: 27190\n", + "Upserted 809 vectors, total so far: 27999\n", + "Upserted 810 vectors, total so far: 28809\n", + "Upserted 805 vectors, total so far: 29614\n", + "Upserted 803 vectors, total so far: 30417\n", + "Upserted 790 vectors, total so far: 31207\n", + "Upserted 818 vectors, total so far: 32025\n", + "Upserted 805 vectors, total so far: 32830\n", + "Upserted 790 vectors, total so far: 33620\n", + "Upserted 795 vectors, total so far: 34415\n", + "Upserted 801 vectors, total so far: 35216\n", + "Upserted 789 vectors, total so far: 36005\n", + "Upserted 810 vectors, total so far: 36815\n", + "Upserted 796 vectors, total so far: 37611\n", + "Upserted 791 vectors, total so far: 38402\n", + "Upserted 785 vectors, total so far: 39187\n", + "Upserted 820 vectors, total so far: 40007\n", + "Upserted 812 vectors, total so far: 40819\n", + "Upserted 789 vectors, total so far: 41608\n", + "Upserted 796 vectors, total so far: 42404\n", + "Upserted 807 vectors, total so far: 43211\n", + "Upserted 805 vectors, total so far: 44016\n", + "Upserted 792 vectors, total so far: 44808\n", + "Upserted 788 vectors, total so far: 45596\n", + "Upserted 785 vectors, total so far: 46381\n", + "Upserted 818 vectors, total so far: 47199\n", + "Upserted 807 vectors, total so far: 48006\n", + "Upserted 818 vectors, total so far: 48824\n", + "Upserted 790 vectors, total so far: 49614\n", + "Upserted 797 vectors, total so far: 50411\n", + "Upserted 816 vectors, total so far: 51227\n", + "Upserted 795 vectors, total so far: 52022\n", + "Upserted 807 vectors, total so far: 52829\n", + "Upserted 778 vectors, total so far: 53607\n", + "Upserted 785 vectors, total so far: 54392\n", + "Upserted 807 vectors, total so far: 55199\n", + "Upserted 792 vectors, total so far: 55991\n", + "Upserted 822 vectors, total so far: 56813\n", + "Upserted 785 vectors, total so far: 57598\n", + "Upserted 789 vectors, total so far: 58387\n", + "Upserted 807 vectors, total so far: 59194\n", + "Upserted 811 vectors, total so far: 60005\n", + "Upserted 801 vectors, total so far: 60806\n", + "Upserted 780 vectors, total so far: 61586\n", + "Upserted 806 vectors, total so far: 62392\n", + "Upserted 791 vectors, total so far: 63183\n", + "Upserted 787 vectors, total so far: 63970\n", + "Upserted 794 vectors, total so far: 64764\n" + ] + }, + { + "ename": "ConnectionError", + "evalue": "HTTPConnectionPool(host='148.113.54.173', port=8080): Max retries exceeded with url: /api/v1/index/test_1M_vaib_reupsertcheck_new1/vector/insert (Caused by NewConnectionError(\"HTTPConnection(host='148.113.54.173', port=8080): Failed to establish a new connection: [Errno 111] Connection refused\"))", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mConnectionRefusedError\u001b[39m Traceback (most recent call last)", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connection.py:204\u001b[39m, in \u001b[36mHTTPConnection._new_conn\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 203\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m204\u001b[39m sock = \u001b[43mconnection\u001b[49m\u001b[43m.\u001b[49m\u001b[43mcreate_connection\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 205\u001b[39m \u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_dns_host\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mport\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 206\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 207\u001b[39m \u001b[43m \u001b[49m\u001b[43msource_address\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msource_address\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 208\u001b[39m \u001b[43m \u001b[49m\u001b[43msocket_options\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msocket_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 209\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 210\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m socket.gaierror \u001b[38;5;28;01mas\u001b[39;00m e:\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/util/connection.py:85\u001b[39m, in \u001b[36mcreate_connection\u001b[39m\u001b[34m(address, timeout, source_address, socket_options)\u001b[39m\n\u001b[32m 84\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m---> \u001b[39m\u001b[32m85\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m err\n\u001b[32m 86\u001b[39m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[32m 87\u001b[39m \u001b[38;5;66;03m# Break explicitly a reference cycle\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/util/connection.py:73\u001b[39m, in \u001b[36mcreate_connection\u001b[39m\u001b[34m(address, timeout, source_address, socket_options)\u001b[39m\n\u001b[32m 72\u001b[39m sock.bind(source_address)\n\u001b[32m---> \u001b[39m\u001b[32m73\u001b[39m \u001b[43msock\u001b[49m\u001b[43m.\u001b[49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43msa\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 74\u001b[39m \u001b[38;5;66;03m# Break explicitly a reference cycle\u001b[39;00m\n", + "\u001b[31mConnectionRefusedError\u001b[39m: [Errno 111] Connection refused", + "\nThe above exception was the direct cause of the following exception:\n", + "\u001b[31mNewConnectionError\u001b[39m Traceback (most recent call last)", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connectionpool.py:787\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 786\u001b[39m \u001b[38;5;66;03m# Make the request on the HTTPConnection object\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m787\u001b[39m response = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_make_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 788\u001b[39m \u001b[43m \u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 789\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 790\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 791\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 792\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 793\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 794\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 795\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m=\u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 796\u001b[39m \u001b[43m \u001b[49m\u001b[43mresponse_conn\u001b[49m\u001b[43m=\u001b[49m\u001b[43mresponse_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 797\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 798\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 799\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 800\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 802\u001b[39m \u001b[38;5;66;03m# Everything went great!\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connectionpool.py:493\u001b[39m, in \u001b[36mHTTPConnectionPool._make_request\u001b[39m\u001b[34m(self, conn, method, url, body, headers, retries, timeout, chunked, response_conn, preload_content, decode_content, enforce_content_length)\u001b[39m\n\u001b[32m 492\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m493\u001b[39m \u001b[43mconn\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 494\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 495\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 496\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 497\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 498\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 499\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 500\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 501\u001b[39m \u001b[43m \u001b[49m\u001b[43menforce_content_length\u001b[49m\u001b[43m=\u001b[49m\u001b[43menforce_content_length\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 502\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 504\u001b[39m \u001b[38;5;66;03m# We are swallowing BrokenPipeError (errno.EPIPE) since the server is\u001b[39;00m\n\u001b[32m 505\u001b[39m \u001b[38;5;66;03m# legitimately able to close the connection after sending a valid response.\u001b[39;00m\n\u001b[32m 506\u001b[39m \u001b[38;5;66;03m# With this behaviour, the received response is still readable.\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connection.py:500\u001b[39m, in \u001b[36mHTTPConnection.request\u001b[39m\u001b[34m(self, method, url, body, headers, chunked, preload_content, decode_content, enforce_content_length)\u001b[39m\n\u001b[32m 499\u001b[39m \u001b[38;5;28mself\u001b[39m.putheader(header, value)\n\u001b[32m--> \u001b[39m\u001b[32m500\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mendheaders\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 502\u001b[39m \u001b[38;5;66;03m# If we're given a body we start sending that in chunks.\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/http/client.py:1298\u001b[39m, in \u001b[36mHTTPConnection.endheaders\u001b[39m\u001b[34m(self, message_body, encode_chunked)\u001b[39m\n\u001b[32m 1297\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m CannotSendHeader()\n\u001b[32m-> \u001b[39m\u001b[32m1298\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_send_output\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmessage_body\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/http/client.py:1058\u001b[39m, in \u001b[36mHTTPConnection._send_output\u001b[39m\u001b[34m(self, message_body, encode_chunked)\u001b[39m\n\u001b[32m 1057\u001b[39m \u001b[38;5;28;01mdel\u001b[39;00m \u001b[38;5;28mself\u001b[39m._buffer[:]\n\u001b[32m-> \u001b[39m\u001b[32m1058\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmsg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1060\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m message_body \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m 1061\u001b[39m \n\u001b[32m 1062\u001b[39m \u001b[38;5;66;03m# create a consistent interface to message_body\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.11/http/client.py:996\u001b[39m, in \u001b[36mHTTPConnection.send\u001b[39m\u001b[34m(self, data)\u001b[39m\n\u001b[32m 995\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.auto_open:\n\u001b[32m--> \u001b[39m\u001b[32m996\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 997\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connection.py:331\u001b[39m, in \u001b[36mHTTPConnection.connect\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 330\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mconnect\u001b[39m(\u001b[38;5;28mself\u001b[39m) -> \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m331\u001b[39m \u001b[38;5;28mself\u001b[39m.sock = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_new_conn\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 332\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m._tunnel_host:\n\u001b[32m 333\u001b[39m \u001b[38;5;66;03m# If we're tunneling it means we're connected to our proxy.\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connection.py:219\u001b[39m, in \u001b[36mHTTPConnection._new_conn\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 218\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m--> \u001b[39m\u001b[32m219\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m NewConnectionError(\n\u001b[32m 220\u001b[39m \u001b[38;5;28mself\u001b[39m, \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mFailed to establish a new connection: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m\n\u001b[32m 221\u001b[39m ) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01me\u001b[39;00m\n\u001b[32m 223\u001b[39m sys.audit(\u001b[33m\"\u001b[39m\u001b[33mhttp.client.connect\u001b[39m\u001b[33m\"\u001b[39m, \u001b[38;5;28mself\u001b[39m, \u001b[38;5;28mself\u001b[39m.host, \u001b[38;5;28mself\u001b[39m.port)\n", + "\u001b[31mNewConnectionError\u001b[39m: HTTPConnection(host='148.113.54.173', port=8080): Failed to establish a new connection: [Errno 111] Connection refused", + "\nThe above exception was the direct cause of the following exception:\n", + "\u001b[31mMaxRetryError\u001b[39m Traceback (most recent call last)", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/requests/adapters.py:644\u001b[39m, in \u001b[36mHTTPAdapter.send\u001b[39m\u001b[34m(self, request, stream, timeout, verify, cert, proxies)\u001b[39m\n\u001b[32m 643\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m644\u001b[39m resp = \u001b[43mconn\u001b[49m\u001b[43m.\u001b[49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 645\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 646\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m=\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 647\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 648\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 649\u001b[39m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 650\u001b[39m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 651\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 652\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 653\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mmax_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 654\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 655\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 656\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 658\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m (ProtocolError, \u001b[38;5;167;01mOSError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m err:\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connectionpool.py:871\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 868\u001b[39m log.warning(\n\u001b[32m 869\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mRetrying (\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m) after connection broken by \u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[33m\"\u001b[39m, retries, err, url\n\u001b[32m 870\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m871\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 872\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 873\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 874\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 875\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 876\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 877\u001b[39m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 878\u001b[39m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 879\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 880\u001b[39m \u001b[43m \u001b[49m\u001b[43mpool_timeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpool_timeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 881\u001b[39m \u001b[43m \u001b[49m\u001b[43mrelease_conn\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrelease_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 882\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 883\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody_pos\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbody_pos\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 884\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 885\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 886\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 887\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 889\u001b[39m \u001b[38;5;66;03m# Handle redirect?\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connectionpool.py:871\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 868\u001b[39m log.warning(\n\u001b[32m 869\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mRetrying (\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m) after connection broken by \u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[33m\"\u001b[39m, retries, err, url\n\u001b[32m 870\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m871\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 872\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 873\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 874\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 875\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 876\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 877\u001b[39m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 878\u001b[39m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 879\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 880\u001b[39m \u001b[43m \u001b[49m\u001b[43mpool_timeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpool_timeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 881\u001b[39m \u001b[43m \u001b[49m\u001b[43mrelease_conn\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrelease_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 882\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 883\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody_pos\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbody_pos\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 884\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 885\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 886\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 887\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 889\u001b[39m \u001b[38;5;66;03m# Handle redirect?\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connectionpool.py:871\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 868\u001b[39m log.warning(\n\u001b[32m 869\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mRetrying (\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m) after connection broken by \u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[33m\"\u001b[39m, retries, err, url\n\u001b[32m 870\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m871\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 872\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 873\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 874\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 875\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 876\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 877\u001b[39m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 878\u001b[39m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 879\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 880\u001b[39m \u001b[43m \u001b[49m\u001b[43mpool_timeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpool_timeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 881\u001b[39m \u001b[43m \u001b[49m\u001b[43mrelease_conn\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrelease_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 882\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 883\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody_pos\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbody_pos\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 884\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 885\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 886\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 887\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 889\u001b[39m \u001b[38;5;66;03m# Handle redirect?\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/connectionpool.py:841\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 839\u001b[39m new_e = ProtocolError(\u001b[33m\"\u001b[39m\u001b[33mConnection aborted.\u001b[39m\u001b[33m\"\u001b[39m, new_e)\n\u001b[32m--> \u001b[39m\u001b[32m841\u001b[39m retries = \u001b[43mretries\u001b[49m\u001b[43m.\u001b[49m\u001b[43mincrement\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 842\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43merror\u001b[49m\u001b[43m=\u001b[49m\u001b[43mnew_e\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_pool\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_stacktrace\u001b[49m\u001b[43m=\u001b[49m\u001b[43msys\u001b[49m\u001b[43m.\u001b[49m\u001b[43mexc_info\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[32;43m2\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[32m 843\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 844\u001b[39m retries.sleep()\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/urllib3/util/retry.py:535\u001b[39m, in \u001b[36mRetry.increment\u001b[39m\u001b[34m(self, method, url, response, error, _pool, _stacktrace)\u001b[39m\n\u001b[32m 534\u001b[39m reason = error \u001b[38;5;129;01mor\u001b[39;00m ResponseError(cause)\n\u001b[32m--> \u001b[39m\u001b[32m535\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m MaxRetryError(_pool, url, reason) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mreason\u001b[39;00m \u001b[38;5;66;03m# type: ignore[arg-type]\u001b[39;00m\n\u001b[32m 537\u001b[39m log.debug(\u001b[33m\"\u001b[39m\u001b[33mIncremented Retry for (url=\u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m): \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m\"\u001b[39m, url, new_retry)\n", + "\u001b[31mMaxRetryError\u001b[39m: HTTPConnectionPool(host='148.113.54.173', port=8080): Max retries exceeded with url: /api/v1/index/test_1M_vaib_reupsertcheck_new1/vector/insert (Caused by NewConnectionError(\"HTTPConnection(host='148.113.54.173', port=8080): Failed to establish a new connection: [Errno 111] Connection refused\"))", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[31mConnectionError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[8]\u001b[39m\u001b[32m, line 52\u001b[39m\n\u001b[32m 42\u001b[39m vector_data = {\n\u001b[32m 43\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mid\u001b[39m\u001b[33m\"\u001b[39m: \u001b[38;5;28mstr\u001b[39m(row[\u001b[33m\"\u001b[39m\u001b[33mid\u001b[39m\u001b[33m\"\u001b[39m]),\n\u001b[32m 44\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mvector\u001b[39m\u001b[33m\"\u001b[39m: row[\u001b[33m\"\u001b[39m\u001b[33memb\u001b[39m\u001b[33m\"\u001b[39m],\n\u001b[32m (...)\u001b[39m\u001b[32m 48\u001b[39m }\n\u001b[32m 49\u001b[39m }\n\u001b[32m 50\u001b[39m batch_vectors.append(vector_data)\n\u001b[32m---> \u001b[39m\u001b[32m52\u001b[39m \u001b[43mindex\u001b[49m\u001b[43m.\u001b[49m\u001b[43mupsert\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbatch_vectors\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 53\u001b[39m total_inserted += \u001b[38;5;28mlen\u001b[39m(batch_vectors)\n\u001b[32m 54\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mUpserted \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(batch_vectors)\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m vectors, total so far: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtotal_inserted\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m)\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/endee/index.py:338\u001b[39m, in \u001b[36mIndex.upsert\u001b[39m\u001b[34m(self, input_array)\u001b[39m\n\u001b[32m 335\u001b[39m http_client = \u001b[38;5;28mself\u001b[39m._get_session_client()\n\u001b[32m 337\u001b[39m \u001b[38;5;66;03m# Sending the batch to the server\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m338\u001b[39m response = \u001b[43mhttp_client\u001b[49m\u001b[43m.\u001b[49m\u001b[43mpost\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 339\u001b[39m \u001b[43m \u001b[49m\u001b[33;43mf\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[33;43m/index/\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mname\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[33;43m/vector/insert\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 340\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 341\u001b[39m \u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m=\u001b[49m\u001b[43mserialized_data\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 342\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 344\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m response.status_code != \u001b[32m200\u001b[39m:\n\u001b[32m 345\u001b[39m raise_exception(response.status_code, response.text)\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/requests/sessions.py:637\u001b[39m, in \u001b[36mSession.post\u001b[39m\u001b[34m(self, url, data, json, **kwargs)\u001b[39m\n\u001b[32m 626\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mpost\u001b[39m(\u001b[38;5;28mself\u001b[39m, url, data=\u001b[38;5;28;01mNone\u001b[39;00m, json=\u001b[38;5;28;01mNone\u001b[39;00m, **kwargs):\n\u001b[32m 627\u001b[39m \u001b[38;5;250m \u001b[39m\u001b[33mr\u001b[39m\u001b[33;03m\"\"\"Sends a POST request. Returns :class:`Response` object.\u001b[39;00m\n\u001b[32m 628\u001b[39m \n\u001b[32m 629\u001b[39m \u001b[33;03m :param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 634\u001b[39m \u001b[33;03m :rtype: requests.Response\u001b[39;00m\n\u001b[32m 635\u001b[39m \u001b[33;03m \"\"\"\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m637\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mPOST\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mjson\u001b[49m\u001b[43m=\u001b[49m\u001b[43mjson\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/requests/sessions.py:589\u001b[39m, in \u001b[36mSession.request\u001b[39m\u001b[34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[39m\n\u001b[32m 584\u001b[39m send_kwargs = {\n\u001b[32m 585\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mtimeout\u001b[39m\u001b[33m\"\u001b[39m: timeout,\n\u001b[32m 586\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mallow_redirects\u001b[39m\u001b[33m\"\u001b[39m: allow_redirects,\n\u001b[32m 587\u001b[39m }\n\u001b[32m 588\u001b[39m send_kwargs.update(settings)\n\u001b[32m--> \u001b[39m\u001b[32m589\u001b[39m resp = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprep\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43msend_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 591\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m resp\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/requests/sessions.py:703\u001b[39m, in \u001b[36mSession.send\u001b[39m\u001b[34m(self, request, **kwargs)\u001b[39m\n\u001b[32m 700\u001b[39m start = preferred_clock()\n\u001b[32m 702\u001b[39m \u001b[38;5;66;03m# Send the request\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m703\u001b[39m r = \u001b[43madapter\u001b[49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 705\u001b[39m \u001b[38;5;66;03m# Total elapsed time of the request (approximately)\u001b[39;00m\n\u001b[32m 706\u001b[39m elapsed = preferred_clock() - start\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/latest_VDB/VectorDBBench/venv2/lib/python3.11/site-packages/requests/adapters.py:677\u001b[39m, in \u001b[36mHTTPAdapter.send\u001b[39m\u001b[34m(self, request, stream, timeout, verify, cert, proxies)\u001b[39m\n\u001b[32m 673\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(e.reason, _SSLError):\n\u001b[32m 674\u001b[39m \u001b[38;5;66;03m# This branch is for urllib3 v1.22 and later.\u001b[39;00m\n\u001b[32m 675\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m SSLError(e, request=request)\n\u001b[32m--> \u001b[39m\u001b[32m677\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(e, request=request)\n\u001b[32m 679\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m ClosedPoolError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m 680\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(e, request=request)\n", + "\u001b[31mConnectionError\u001b[39m: HTTPConnectionPool(host='148.113.54.173', port=8080): Max retries exceeded with url: /api/v1/index/test_1M_vaib_reupsertcheck_new1/vector/insert (Caused by NewConnectionError(\"HTTPConnection(host='148.113.54.173', port=8080): Failed to establish a new connection: [Errno 111] Connection refused\"))" + ] + } + ], + "source": [ + "#For reinserting deleted vectors (int filter)\n", + "import pyarrow.parquet as pq\n", + "import os\n", + "import time\n", + "\n", + "# User inputs\n", + "int_rate_insert = \"80p\" # change to 1p, 50p, 80p, 99p\n", + "batch_size = 1000\n", + "\n", + "# Map int_rate to range\n", + "range_map = {\n", + " \"1p\": [0, 9999],\n", + " \"50p\": [0, 499999],\n", + " \"60p\": [0, 599999],\n", + " \"70p\": [0, 699999],\n", + " \"80p\": [0, 799999],\n", + " \"99p\": [0, 989999],\n", + "}\n", + "filter_range = range_map[int_rate_insert]\n", + "low, high = filter_range\n", + "\n", + "# Build full paths\n", + "dataset_path = os.path.join(base_path, dataset_folder)\n", + "emb_path = os.path.join(dataset_path, \"shuffle_train.parquet\")\n", + "\n", + "# Process shuffle_train in batches to avoid RAM issues\n", + "emb_file = pq.ParquetFile(emb_path)\n", + "total_inserted = 0\n", + "\n", + "start = time.perf_counter()\n", + "for batch in emb_file.iter_batches(batch_size=batch_size):\n", + " batch_df = batch.to_pandas()\n", + "\n", + " # Filter by id range\n", + " batch_df = batch_df[(batch_df[\"id\"] >= low) & (batch_df[\"id\"] <= high)]\n", + "\n", + " if batch_df.empty:\n", + " continue\n", + "\n", + " batch_vectors = []\n", + " for _, row in batch_df.iterrows():\n", + " vector_data = {\n", + " \"id\": str(row[\"id\"]),\n", + " \"vector\": row[\"emb\"],\n", + " \"meta\": {\"id\": row[\"id\"]},\n", + " \"filter\": {\n", + " \"id\": row[\"id\"],\n", + " }\n", + " }\n", + " batch_vectors.append(vector_data)\n", + "\n", + " index.upsert(batch_vectors)\n", + " total_inserted += len(batch_vectors)\n", + " print(f\"Upserted {len(batch_vectors)} vectors, total so far: {total_inserted}\")\n", + "end = time.perf_counter()\n", + "print(f\"Done! Total inserted: {total_inserted} vectors with id range {filter_range}\")\n", + "print(\"Time taken:\", end-start)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "index.describe()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv2", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/script.ipynb b/script.ipynb new file mode 100644 index 000000000..520dc89c0 --- /dev/null +++ b/script.ipynb @@ -0,0 +1,329 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pip install pandas pyarrow" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pip install endee" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pip show endee" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "base_path = \"/home/debian/latest_VDB/VectorDBBench/vectordataset_label\"\n", + "dataset_folder = \"cohere/cohere_medium_1m\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#For checking parquet file contents\n", + "import pyarrow.parquet as pq\n", + "import os\n", + "\n", + "file_name = \"shuffle_train.parquet\" #input\n", + "\n", + "# Build full path\n", + "file_path = os.path.join(base_path, dataset_folder, file_name)\n", + "\n", + "\n", + "parquet_file = pq.ParquetFile(file_path)\n", + "\n", + "# Read only first batch of rows\n", + "first_batch = next(parquet_file.iter_batches(batch_size=5))\n", + "preview = first_batch.to_pandas()\n", + "\n", + "for col in preview.columns:\n", + " if preview[col].dtype == object and isinstance(preview[col].iloc[0], list):\n", + " preview[col] = preview[col].apply(lambda x: x[:5] if x is not None else x)\n", + "\n", + "print(preview)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from endee import Endee\n", + "client = Endee(token=\"localtest\")\n", + "client.set_base_url(\"http://148.113.54.173:8080/api/v1\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#For checking indexes\n", + "for obj in client.list_indexes().get('indexes'):\n", + " print(obj['name'])\n", + " print(obj['total_elements'])\n", + " print('\\t')" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "#give the index name\n", + "index_name = \"test_1M_labelfilter_int16_latest_master\"\n", + "index = client.get_index(index_name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "index.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#For querying\n", + "import pyarrow.parquet as pq\n", + "import os\n", + "\n", + "#inputs\n", + "label_to_query = \"label_1p\"\n", + "top_k = 30\n", + "\n", + "# Mode 1: Single query id\n", + "# Mode 2: Find all query ids with recall < 1\n", + "mode = 2 # change to 1 or 2\n", + "query_id = 457 # only used in mode 1\n", + "\n", + "# Build full paths\n", + "dataset_path = os.path.join(base_path, dataset_folder)\n", + "test_parquet_path = os.path.join(dataset_path, \"test.parquet\")\n", + "gt_parquet_path = os.path.join(dataset_path, f\"neighbors_labels_{label_to_query}.parquet\")\n", + "\n", + "# Read parquet files\n", + "test_df = pq.ParquetFile(test_parquet_path).read().to_pandas()\n", + "gt_df = pq.ParquetFile(gt_parquet_path).read().to_pandas()\n", + "\n", + "def run_query(query_id):\n", + " query_row = test_df[test_df[\"id\"] == query_id]\n", + " if query_row.empty:\n", + " print(f\"Query ID {query_id} not found in test.parquet\")\n", + " return None\n", + "\n", + " query_vector = query_row[\"emb\"].values[0]\n", + "\n", + " gt_row = gt_df[gt_df[\"id\"] == query_id]\n", + " if gt_row.empty:\n", + " print(f\"Query ID {query_id} not found in ground truth file\")\n", + " return None\n", + "\n", + " ground_truth = gt_row[\"neighbors_id\"].values[0][:top_k]\n", + "\n", + " results = index.query(\n", + " vector=query_vector,\n", + " top_k=top_k,\n", + " filter=[{\"label\": {\"$eq\": label_to_query}}],\n", + " include_vectors=False,\n", + " )\n", + " returned_ids = [int(r[\"id\"]) for r in results]\n", + "\n", + " intersection = len(set(returned_ids) & set(ground_truth))\n", + " recall = intersection / len(ground_truth)\n", + "\n", + " return returned_ids, ground_truth, recall\n", + "\n", + "\n", + "if mode == 1:\n", + " result = run_query(query_id)\n", + " if result:\n", + " returned_ids, ground_truth, recall = result\n", + " print(f\"Query ID: {query_id}\")\n", + " print(f\"Returned IDs: {','.join(map(str, returned_ids))}\")\n", + " print(f\"Ground Truth: {','.join(map(str, ground_truth))}\")\n", + " print(f\"Recall: {recall:.4f}\")\n", + "\n", + "elif mode == 2:\n", + " print(f\"Finding all query IDs with recall < 1.0 ...\\n\")\n", + " low_recall_ids = []\n", + " all_recalls = []\n", + " for query_id in test_df[\"id\"].values:\n", + " result = run_query(query_id)\n", + " if result:\n", + " returned_ids, ground_truth, recall = result\n", + " all_recalls.append(recall)\n", + " if recall < 1.0:\n", + " low_recall_ids.append(query_id)\n", + " print(f\"Query ID: {query_id}\")\n", + " print(f\"Returned IDs: {','.join(map(str, returned_ids))}\")\n", + " print(f\"Ground Truth: {','.join(map(str, ground_truth))}\")\n", + " print(f\"Recall: {recall:.4f}\\n\")\n", + "\n", + " print(f\"Total query IDs with recall < 1.0: {len(low_recall_ids)}\")\n", + " print(f\"IDs: {low_recall_ids}\")\n", + " print(f\"Average Recall across all {len(all_recalls)} queries: {sum(all_recalls)/len(all_recalls):.4f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "index.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#For deleting\n", + "# Delete by label\n", + "label_to_delete = \"label_1p\" #input\n", + "\n", + "result = index.delete_with_filter([{\"label\": {\"$eq\": label_to_delete}}])\n", + "print(f\"Deleted vectors with label: {label_to_delete}\")\n", + "print(result)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "index.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#For reinserting delted vectors\n", + "import pyarrow.parquet as pq\n", + "import pandas as pd\n", + "import os\n", + "\n", + "# User inputs\n", + "label_to_insert = \"label_1p\"\n", + "batch_size = 1000\n", + "\n", + "# Build full paths\n", + "dataset_path = os.path.join(base_path, dataset_folder)\n", + "labels_path = os.path.join(dataset_path, \"scalar_labels.parquet\")\n", + "emb_path = os.path.join(dataset_path, \"shuffle_train.parquet\")\n", + "\n", + "# Read labels file fully (small file, just id + label)\n", + "labels_df = pq.ParquetFile(labels_path).read().to_pandas()\n", + "\n", + "# Filter only the label we need\n", + "filtered_labels = labels_df[labels_df[\"labels\"] == label_to_insert]\n", + "valid_ids = set(filtered_labels[\"id\"].values)\n", + "\n", + "print(f\"Total vectors to insert: {len(valid_ids)}\")\n", + "\n", + "# Process shuffle_train in batches to avoid RAM issues\n", + "emb_file = pq.ParquetFile(emb_path)\n", + "total_inserted = 0\n", + "\n", + "for batch in emb_file.iter_batches(batch_size=batch_size):\n", + " batch_df = batch.to_pandas()\n", + " \n", + " # Keep only rows whose id is in filtered labels\n", + " batch_df = batch_df[batch_df[\"id\"].isin(valid_ids)]\n", + " \n", + " if batch_df.empty:\n", + " continue\n", + " \n", + " # Merge to get labels\n", + " batch_df = batch_df.merge(filtered_labels[[\"id\", \"labels\"]], on=\"id\")\n", + " \n", + " batch_vectors = []\n", + " for _, row in batch_df.iterrows():\n", + " vector_data = {\n", + " \"id\": str(row[\"id\"]),\n", + " \"vector\": row[\"emb\"],\n", + " \"meta\": {\"id\": row[\"id\"]},\n", + " \"filter\": {\n", + " \"id\": row[\"id\"],\n", + " \"label\": row[\"labels\"]\n", + " }\n", + " }\n", + " batch_vectors.append(vector_data)\n", + " \n", + " index.upsert(batch_vectors)\n", + " total_inserted += len(batch_vectors)\n", + " print(f\"Upserted {len(batch_vectors)} vectors, total so far: {total_inserted}\")\n", + "\n", + "print(f\"Done! Total inserted: {total_inserted} vectors with label '{label_to_insert}'\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "index.describe()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv2", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/vectordb_bench/backend/clients/endee/endee.py b/vectordb_bench/backend/clients/endee/endee.py index dae5d1ed3..6f7d3dea9 100644 --- a/vectordb_bench/backend/clients/endee/endee.py +++ b/vectordb_bench/backend/clients/endee/endee.py @@ -390,15 +390,34 @@ def optimize(self, data_size: int | None = None): # else: # msg = f"Not support Filter for Endee - {filters}" # raise ValueError(msg) + #--------------------------------------------------------------------------- + # def prepare_filter(self, filters: Filter): + # if filters.type == FilterOp.NonFilter: + # self.filter_expr = None + + # elif filters.type == FilterOp.NumGE: + # # Endee supports $range, NOT $gte + # # Dataset size = 1 million → finite upper bound + # self.filter_expr = [ + # {self._scalar_id_field: {"$range": [filters.int_value, 1_000_000]}} + # ] + + # elif filters.type == FilterOp.StrEqual: + # self.filter_expr = [ + # {self._scalar_label_field: {"$eq": filters.label_value}} + # ] + + # else: + # raise ValueError(f"Not support Filter for Endee - {filters}") + #----------------------------------------------------------------------- def prepare_filter(self, filters: Filter): if filters.type == FilterOp.NonFilter: self.filter_expr = None elif filters.type == FilterOp.NumGE: - # Endee supports $range, NOT $gte # Dataset size = 1 million → finite upper bound self.filter_expr = [ - {self._scalar_id_field: {"$range": [filters.int_value, 1_000_000]}} + {self._scalar_id_field: {"$gte": filters.int_value}} ] elif filters.type == FilterOp.StrEqual: @@ -410,6 +429,47 @@ def prepare_filter(self, filters: Filter): raise ValueError(f"Not support Filter for Endee - {filters}") + def insert_embeddings( + self, + embeddings: Iterable[list[float]], + metadata: list[int], + labels_data: list[str] | None = None, + **kwargs, + ) -> tuple[int, Exception]: + """ + Insert embeddings with filter metadata. + Modified to include filter fields like Milvus does. + """ + assert len(embeddings) == len(metadata) + insert_count = 0 + try: + batch_vectors = [] + for i in range(len(embeddings)): + vector_data = { + "id": str(metadata[i]), + "vector": embeddings[i], + "meta": {"id": metadata[i]}, # Store id in meta for reference + "filter": { + self._scalar_id_field: metadata[i] # Store id for numeric filtering + } + } + + # Add label field if using scalar labels + if self.with_scalar_labels and labels_data is not None: + vector_data["filter"][self._scalar_label_field] = labels_data[i] + + batch_vectors.append(vector_data) + + self.index.upsert(batch_vectors) + insert_count = len(batch_vectors) + + except Exception as e: + log.error(f"Failed to insert data: {e}") + return insert_count, e + + return (len(embeddings), None) + + # def insert_embeddings( # self, # embeddings: Iterable[list[float]], @@ -426,6 +486,8 @@ def prepare_filter(self, filters: Filter): # try: # batch_vectors = [] # for i in range(len(embeddings)): + # if labels_data is None or labels_data[i] != "label_1p": + # continue # vector_data = { # "id": str(metadata[i]), # "vector": embeddings[i], @@ -440,9 +502,10 @@ def prepare_filter(self, filters: Filter): # vector_data["filter"][self._scalar_label_field] = labels_data[i] # batch_vectors.append(vector_data) - - # self.index.upsert(batch_vectors) - # insert_count = len(batch_vectors) + + # if batch_vectors != []: + # self.index.upsert(batch_vectors) + # insert_count = len(batch_vectors) # except Exception as e: # log.error(f"Failed to insert data: {e}") @@ -458,35 +521,36 @@ def prepare_filter(self, filters: Filter): # labels_data: list[str] | None = None, # **kwargs, # ) -> tuple[int, Exception]: - # """ - # Insert embeddings with filter metadata. - # Modified to include filter fields like Milvus does. - # """ # assert len(embeddings) == len(metadata) # insert_count = 0 # try: # batch_vectors = [] # for i in range(len(embeddings)): - # if labels_data is None or labels_data[i] != "label_1p": + # if metadata[i] > 9999 or metadata[i] < 0: # continue # vector_data = { # "id": str(metadata[i]), # "vector": embeddings[i], - # "meta": {"id": metadata[i]}, # Store id in meta for reference + # "meta": {"id": metadata[i]}, # "filter": { - # self._scalar_id_field: metadata[i] # Store id for numeric filtering + # self._scalar_id_field: metadata[i] # } # } - # # Add label field if using scalar labels # if self.with_scalar_labels and labels_data is not None: # vector_data["filter"][self._scalar_label_field] = labels_data[i] # batch_vectors.append(vector_data) - + + # # # Log matched metadata IDs to file + # # with open("matched_metadata.txt", "a") as f: + # # for v in batch_vectors: + # # f.write(v["id"] + "\n") + # if batch_vectors != []: # self.index.upsert(batch_vectors) # insert_count = len(batch_vectors) + # # time.sleep(20) # except Exception as e: # log.error(f"Failed to insert data: {e}") @@ -494,51 +558,6 @@ def prepare_filter(self, filters: Filter): # return (len(embeddings), None) - - def insert_embeddings( - self, - embeddings: Iterable[list[float]], - metadata: list[int], - labels_data: list[str] | None = None, - **kwargs, - ) -> tuple[int, Exception]: - assert len(embeddings) == len(metadata) - insert_count = 0 - try: - batch_vectors = [] - for i in range(len(embeddings)): - if metadata[i] > 9999 or metadata[i] < 0: - continue - vector_data = { - "id": str(metadata[i]), - "vector": embeddings[i], - "meta": {"id": metadata[i]}, - "filter": { - self._scalar_id_field: metadata[i] - } - } - - if self.with_scalar_labels and labels_data is not None: - vector_data["filter"][self._scalar_label_field] = labels_data[i] - - batch_vectors.append(vector_data) - - # # Log matched metadata IDs to file - # with open("matched_metadata.txt", "a") as f: - # for v in batch_vectors: - # f.write(v["id"] + "\n") - - if batch_vectors != []: - self.index.upsert(batch_vectors) - insert_count = len(batch_vectors) - # time.sleep(20) - - except Exception as e: - log.error(f"Failed to insert data: {e}") - return insert_count, e - - return (len(embeddings), None) - def search_embedding( self, query: list[float], From ae4ef174ad6f291493cb36e6d75e171e0de11e66 Mon Sep 17 00:00:00 2001 From: darshan-endee Date: Fri, 27 Feb 2026 09:21:25 +0000 Subject: [PATCH 19/21] sh file added --- run.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 run.sh diff --git a/run.sh b/run.sh new file mode 100644 index 000000000..35e33f4e4 --- /dev/null +++ b/run.sh @@ -0,0 +1,23 @@ + DATASET_LOCAL_DIR="/home/debian/latest_VDB/VectorDBBench/vectordataset" vectordbbench endee \ + --token "localtest" \ + --region india-west-1 \ + --base-url "http://148.113.54.173:8080/api/v1" \ + --index-name test_1M_vaib_reupsertcheck \ + --task-label "20260107" \ + --m 16 \ + --ef-con 128 \ + --ef-search 128 \ + --space-type cosine \ + --precision int16d \ + --version 1 \ + --case-type NewIntFilterPerformanceCase \ + --dataset-with-size-type "Medium Cohere (768dim, 1M)" \ + --filter-rate 0.50 \ + --k 30 \ + --num-concurrency "16" \ + --concurrency-duration 30 \ + --concurrency-timeout 3600 \ + --drop-old \ + --load \ + --search-concurrent \ + --search-serial \ No newline at end of file From ec69debb1ab6a5d2cd11daf8556221169ade0522 Mon Sep 17 00:00:00 2001 From: darshan-endee Date: Fri, 27 Feb 2026 11:00:50 +0000 Subject: [PATCH 20/21] final_intfilterscript added --- intfilterscript_final.ipynb | 337 ++++++++++++++++++++++++++++++++++++ 1 file changed, 337 insertions(+) create mode 100644 intfilterscript_final.ipynb diff --git a/intfilterscript_final.ipynb b/intfilterscript_final.ipynb new file mode 100644 index 000000000..cb399efe4 --- /dev/null +++ b/intfilterscript_final.ipynb @@ -0,0 +1,337 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pip install pandas pyarrow" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pip install endee" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pip show endee" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "base_path = \"/home/debian/latest_VDB/VectorDBBench/vectordataset\"\n", + "dataset_folder = \"cohere/cohere_medium_1m\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#For checking parquet file contents\n", + "import pyarrow.parquet as pq\n", + "import os\n", + "\n", + "file_name = \"shuffle_train.parquet\" #input\n", + "\n", + "# Build full path\n", + "file_path = os.path.join(base_path, dataset_folder, file_name)\n", + "\n", + "\n", + "parquet_file = pq.ParquetFile(file_path)\n", + "\n", + "# Read only first batch of rows\n", + "first_batch = next(parquet_file.iter_batches(batch_size=5))\n", + "preview = first_batch.to_pandas()\n", + "\n", + "for col in preview.columns:\n", + " if preview[col].dtype == object and isinstance(preview[col].iloc[0], list):\n", + " preview[col] = preview[col].apply(lambda x: x[:5] if x is not None else x)\n", + "\n", + "print(preview)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from endee import Endee\n", + "client = Endee(token=\"localtest\")\n", + "client.set_base_url(\"http://148.113.54.173:8080/api/v1\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#For checking indexes\n", + "for obj in client.list_indexes().get('indexes'):\n", + " print(obj['name'])\n", + " print(obj['total_elements'])\n", + " print('\\t')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#give the index name\n", + "index_name = \"test_1M_vaib_reupsertcheck_new1\"\n", + "index = client.get_index(index_name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "index.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#For querying (int filter)- gte filter\n", + "import pyarrow.parquet as pq\n", + "import os\n", + "\n", + "#inputs\n", + "int_rate_query = \"99p\" # change to 1p, 50p, 80p, 99p\n", + "top_k = 30\n", + "\n", + "# Mode 1: Single query id\n", + "# Mode 2: Find all query ids with recall < 1\n", + "mode = 2\n", + "query_id = 457 # only used in mode 1\n", + "\n", + "# Map int_rate_query to filter range\n", + "range_map = {\n", + " \"1p\": [10000, 1000000],\n", + " \"50p\": [500000, 1000000],\n", + " \"80p\": [800000, 1000000],\n", + " \"99p\": [990000, 1000000],\n", + "}\n", + "filter_range = range_map[int_rate_query]\n", + "\n", + "# Build full paths\n", + "dataset_path = os.path.join(base_path, dataset_folder)\n", + "test_parquet_path = os.path.join(dataset_path, \"test.parquet\")\n", + "gt_parquet_path = os.path.join(dataset_path, f\"neighbors_int_{int_rate_query}.parquet\")\n", + "\n", + "# Read parquet files\n", + "test_df = pq.ParquetFile(test_parquet_path).read().to_pandas()\n", + "gt_df = pq.ParquetFile(gt_parquet_path).read().to_pandas()\n", + "\n", + "def run_query(query_id):\n", + " query_row = test_df[test_df[\"id\"] == query_id]\n", + " if query_row.empty:\n", + " print(f\"Query ID {query_id} not found in test.parquet\")\n", + " return None\n", + "\n", + " query_vector = query_row[\"emb\"].values[0]\n", + "\n", + " gt_row = gt_df[gt_df[\"id\"] == query_id]\n", + " if gt_row.empty:\n", + " print(f\"Query ID {query_id} not found in ground truth file\")\n", + " return None\n", + "\n", + " ground_truth = gt_row[\"neighbors_id\"].values[0][:top_k]\n", + "\n", + " results = index.query(\n", + " vector=query_vector,\n", + " top_k=top_k,\n", + " filter=[{\"id\": {\"$gte\": filter_range[0]}}],\n", + " include_vectors=False,\n", + " )\n", + " returned_ids = [int(r[\"id\"]) for r in results]\n", + "\n", + " intersection = len(set(returned_ids) & set(ground_truth))\n", + " recall = intersection / len(ground_truth)\n", + "\n", + " return returned_ids, ground_truth, recall\n", + "\n", + "\n", + "if mode == 1:\n", + " result = run_query(query_id)\n", + " if result:\n", + " returned_ids, ground_truth, recall = result\n", + " print(f\"Query ID: {query_id}\")\n", + " print(f\"Returned IDs: {','.join(map(str, returned_ids))}\")\n", + " print(f\"Ground Truth: {','.join(map(str, ground_truth))}\")\n", + " print(f\"Recall: {recall:.4f}\")\n", + "\n", + "elif mode == 2:\n", + " print(f\"Finding all query IDs with recall < 1.0 ...\\n\")\n", + " low_recall_ids = []\n", + " all_recalls = []\n", + "\n", + " for qid in test_df[\"id\"].values:\n", + " result = run_query(qid)\n", + " if result:\n", + " returned_ids, ground_truth, recall = result\n", + " all_recalls.append(recall)\n", + " if recall < 1.0:\n", + " low_recall_ids.append(qid)\n", + " print(f\"Query ID: {qid}\")\n", + " print(f\"Returned IDs: {','.join(map(str, returned_ids))}\")\n", + " print(f\"Ground Truth: {','.join(map(str, ground_truth))}\")\n", + " print(f\"Recall: {recall:.4f}\\n\")\n", + "\n", + " print(f\"Total query IDs with recall < 1.0: {len(low_recall_ids)}\")\n", + " print(f\"IDs: {low_recall_ids}\")\n", + " print(f\"Average Recall across all {len(all_recalls)} queries: {sum(all_recalls)/len(all_recalls):.4f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "index.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#For deleting (int filter) - lt\n", + "int_rate_delete = \"99p\" # change to 1p, 50p, 80p, 99p\n", + "\n", + "# Map int_rate to lower bound (gte value)\n", + "range_map = {\n", + " \"1p\": 10000,\n", + " \"50p\": 500000,\n", + " \"80p\": 800000,\n", + " \"99p\": 990000,\n", + "}\n", + "gte_value = range_map[int_rate_delete]\n", + "\n", + "result = index.delete_with_filter([{\"id\": {\"$lt\": gte_value}}])\n", + "print(f\"Deleted vectors with id < {gte_value}\")\n", + "print(result)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "index.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#For reinserting deleted vectors (int filter)\n", + "import pyarrow.parquet as pq\n", + "import os\n", + "import time\n", + "\n", + "# User inputs\n", + "int_rate_insert = \"99p\" # change to 1p, 50p, 80p, 99p\n", + "batch_size = 1000\n", + "\n", + "# Map int_rate to range\n", + "range_map = {\n", + " \"1p\": [0, 9999],\n", + " \"50p\": [0, 499999],\n", + " \"80p\": [0, 799999],\n", + " \"99p\": [0, 989999],\n", + "}\n", + "filter_range = range_map[int_rate_insert]\n", + "low, high = filter_range\n", + "\n", + "# Build full paths\n", + "dataset_path = os.path.join(base_path, dataset_folder)\n", + "emb_path = os.path.join(dataset_path, \"shuffle_train.parquet\")\n", + "\n", + "# Process shuffle_train in batches to avoid RAM issues\n", + "emb_file = pq.ParquetFile(emb_path)\n", + "total_inserted = 0\n", + "\n", + "start = time.perf_counter()\n", + "for batch in emb_file.iter_batches(batch_size=batch_size):\n", + " batch_df = batch.to_pandas()\n", + "\n", + " # Filter by id range\n", + " batch_df = batch_df[(batch_df[\"id\"] >= low) & (batch_df[\"id\"] <= high)]\n", + "\n", + " if batch_df.empty:\n", + " continue\n", + "\n", + " batch_vectors = []\n", + " for _, row in batch_df.iterrows():\n", + " vector_data = {\n", + " \"id\": str(row[\"id\"]),\n", + " \"vector\": row[\"emb\"],\n", + " \"meta\": {\"id\": row[\"id\"]},\n", + " \"filter\": {\n", + " \"id\": row[\"id\"],\n", + " }\n", + " }\n", + " batch_vectors.append(vector_data)\n", + "\n", + " index.upsert(batch_vectors)\n", + " total_inserted += len(batch_vectors)\n", + " print(f\"Upserted {len(batch_vectors)} vectors, total so far: {total_inserted}\")\n", + "end = time.perf_counter()\n", + "print(f\"Done! Total inserted: {total_inserted} vectors with id range {filter_range}\")\n", + "print(\"Time taken:\", end-start)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "index.describe()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 26d939ec60225d2b78ef6998530e2901cf5dc6d2 Mon Sep 17 00:00:00 2001 From: darshan-endee Date: Fri, 27 Feb 2026 11:31:26 +0000 Subject: [PATCH 21/21] 0.13b1 version added --- endee-0.1.13b1-py3-none-any.whl | Bin 0 -> 28543 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 endee-0.1.13b1-py3-none-any.whl diff --git a/endee-0.1.13b1-py3-none-any.whl b/endee-0.1.13b1-py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..82625a03a05d77eebf173f89c54ab8424497913a GIT binary patch literal 28543 zcmYhhV~j3Lu&zC}%{8`d+qUgzY}>YN+qP|cW{vGN-+R83?0u51AN{AQyHiQuxvx@` z0R=+?0s?{pvQ**Jm14F%Wc+W({a+ye7iIucGcyJQ11o@)i-7^XgQpD5)V$h)%-lQ; zz0?d1mHNaC-2?-*5)`Gh+#K!P%>1!5D2?Q-jFeocR96tEE|B8?q~FyN(S1_r!8`;8 z0s?>m0sWt%|2Msfy`6)TnX|K%J>b6zsmgrz+l)xNA2bMAv;%A)E-Z?mVmYOPDg|=g z5GfvM49HS}B{>^H|GeW%$fhG%M1w_@-S74wql=K%)!h+#DQphx3(`Oe!(&|?OwfqG zhq>>~>4IxYBsu`la!5x2nLjORzR?R@-6W%SbZp!AArJ5`U>yRWdt{P@Op>Dwut}9>?rE{bIBeHnFR}`z!)8pV!Ve$Y?Scv-2NE*j}cT zj5%wQ;W+U`&MDAe`fF$41&9e3--xpECph;fTG#a>Xh5sB-0rWN(^2V#mQ-U%(CCWh zjOQZZUB7;Bnnm0_t%(Q*3rM#pUX|sqX?_hB#GDfZ?CRAV#nrSEj-)+tB5Lo~wT5!W zQ6z9qdGlBy??Yrd#~$IRTrq2f>CeP#!{#y^1Nk%HedK-QI5-2P89F;-nbRsso%dT+ zb4^J^yiG(EldmgdUWdxQ*vpeo=5L0aP3dDZS~kvVPqs$X@Bg+0mV&ZwP3}u`5EKy5 z5DE~`f9ro*0^sao1aNWwuOV_Zx1F{pkbTEBWSJP0L%R|lxucXl0|Y08v57mtE?PNS zLvcVvoTdl|MuA#R%YVYZjrvR#<`2XtH*LkJqaSrYy*^#y#fuY9(K@R<^r>D|OI*n} z$|y6l%Y_>di7W5(vFqQnXa;j+qk5WZEGf{pvzIG3CUhysnq$avE3_k1)ViB0+PjPD z?yD_L$+aP-Pr6P{pEi;_lU6Hd+FiU!+Fd7Uv`$FV*R@K=WLLvQl+;P@KLoQLQ}8{> zRh$`#LWk?#l?d|CMW$iV3XOei$s*TQ(+tSgvA(6?P>YtdtdAeT> zB3@fQzQ)Pz=`S?OsShAp@}r?#@^ZO>9&#)TaG%W**{neir)@+zb@)c_Zfj(73xHB= z2y9x`?lKnI*OZvmdOo@}__5rq*_WB`ZU{{;FneaZRkz<9(XG*5G0bd6o-~7y^WWEC zTu;jrgGy1wxbv2uMAoN0U2xwGuc87BWH0ti(C9K)M9sQ8Fjwm&*)F0PfM83P^*w6P z`qzZjrixS^l5zPUni<=C1>Psgs{gThor@PU;nD>y_|!d*&%EqTtirmT@W%OFt`z50 zX&169!&Ag5Q=eimnf25~@zqEdpxYEF;fTAnz?e{EbP`$lu>5>w{5bf^36wn_hxzp^ z;$WC5p-UwqZ*V-)uqKdHm&PB}IY-5%VUIl*?A1gioVr2Z2^{6?`Q+h?3}(eviUs>*8ymM(+WgbD#onx6vC^bL@Ec@hjRn zdD-wl1&@CJ8Av7|UoLcLj6^RPY5-;60)mlCnH~3hM+K!G#Nd?*I6l#eb|yGlXic9x zmU1Yw4`XIYw6D&^D3%F1MS|1ublY(BAdxP(tFIZ;uS^1=&$Zdy^Eva2Ie$&Y{kr|u zV_kVTQ$Kezzrg1QL@G&%sgv@8-Vrn%VRPLGHoNQ-g@a-H4jC3x{EetUr|AUpjd(=x)xcR>Qjb9 zfFFx@Fygs=3o^sQV<8ZAHXdS7m$2-!`ljVnyiq(-mv$N1oCAmGa_OUK4%<(=ji=#= z8QOTeO}`*H(ew5;Wjl({>2qu9lMKk9H;|*JIB6i~BZ>3#iJ^G^GqJqx`078E?WeKz zoLXb05Q(U4N|mibq+~<=>K)Xt&$?PFL9sB~kgr?Vf&c}4yFtU6TKd+Ek8Dh!=Y2*m z6|YxJmO2~4+~o4W=TTRmO(gXP{&DT@qGv2+J#qEPtbMn!x~~_^`*>mUMmZ9zTs#eP zEI$7>eXnLeje3u-UM7iT+Fm*Hk_NeiSk2CZ^#wSdD{v0gP2Zl{0jstp+8U*hSb@j~LRsh)x7; zzrsx~1fA{!i7PTWCliaRwP`wtR#_C#&+%5`S(15{;1wOX6tG4*E0udGu$8C*Oy|m4 zE>5^%=-zyt`NN%#_zjp`LDk4V#Z2}7L0*^^R}&eUMB;vz;?i=9i0nEq0k=YSg+Pr& zgvpL&p}glBSXEgaHLm~;Gp2@SYU;dP(Rd28!kOWq9k!QWCcnU_^vnb+a((i{W+cdo zLft93&XiFl6C)8(*_o)h!T3Cu?F2^G=ns-AKBN!(sB1y(1b1bY| z8+fW}x9@h@YJrKs3+UMHKw@i;q?r{+u&6onZ;cDzDVE+OE1jp#hM<&Qknh(;CEXVZR@$sre@y zy{N=kMCuZ(@_=#(W9^H|j4ao1_V22?Gg`ecpIg#oR22tZ-$^-$HVKmhmsd=ttbKSH z4#Lg3rkfjlIafl4v5+pB=NjMhScJJZ5!L)& zz{EUKE8CKAF&YAri!X%fV8!fXfA)#9=X_uxg2nVSm*uM~kH77*;>Y$wGO9IFimk!@ z-NZ6?89SDLeEHt*tm-IW8{q?E(JC3ElZnuORz+}m%&$-opI}vKCJI$5kXP6o)B>?3 zd4c;;!|9HH$$G`frtn^kk1ifB z!4rV)%icOZZCDO3X!g?!=BkE6Cj_B%nH*gOqKYZPUHIxNcm6%PUM|I5{lq?MS^jb5 z%>(ZZ`P^_Q_>PD`!VA=RDqCY+d-oPv2{}NG)fG051`ulDz9Bx1=kOzOLy5fM?%#p( zW?crJc05h^om;#7BK)sn!Te9LbWJxi^ZX}O+~t6P5dKH8{Qpw+KeckhYvX(@?zI0w zL;f6ITGz31OkJ6w<{(D1sK0uNp4gPzp-YK`6~!3{6a(CHWcS)%?)_(=M^LGIZmbSW zhbB=_K|!Iv+oIsmfivS2y7Yur#!5eV+C@uj)s0cJMn#-mY{@hwMeQMlJSEXfZjxNu z83p!&wce^V%2--mjI2-0G4~AdL`1uui*uacqsS2+NlH~>Iznm4@+D==MVKB*Du*p1 zS7~gCI2`-&A`oC0#KbFm!9pc1s5{KUjtv)hBGg3{T}2L#TT^8$qArK<9vy@HL0Fx| z7*&tJ0$>QB;&>Y<-puUq^>lXg!tUb%58K_> z`}=llF3>WQJ%2VKg<0+8;O*>unPkix+V~Xxn-fXVjvlL`+VyZCxDNh_x122CtQu5vm6eo4YqO?g%OwxIX=LUb6!*3(Hjs>Zl~2=^pZ)_0TCUKkHF*u^+?4kBf&h! zL(PZ54xV@Z6j~gb2xpshROWArI4(1G!Y+5!E1fPYlf8#zAmzKZG&cqFyEK%p%brEq zP^ozaPu;Ov0Y98G7B;_+-nU&m45hCR`;EEqFhXm5)8e~PlSrPB zd!7*ti5jTB;M!%yP4b31ZEn1X3=$z#Mdo6v;H43ya^Si;0+n{PiHR@Hp{-0X6duTj z$%2RT`z#%V;|B1Cql1sBtSNAV*h;CcyQl=Y6-?C>u`aVMWqisLi~bbNnT`F=HtD5SYO^ zI8Y2oUV#JUtri^NcyemlRDX*IYmmY%^VSYKtPq>C>co1dx8^#Jvq{o@mKEvS4rlU^&Q>Lh`XbmI52-Eks;1I_40g3h z6R}Pjt1BsoX}L%Il)q_K3O zFY2mtnwCdy1-Ir`yd0R=ZfWf;+@J9gA=pGJYHI{VW!Z_3pCQ-}hn^qa&kdIfG@$%u zG&+gFh+iH!50h`m>iLyRA|UYOUy$IaK`aT|ohAjr;so0x8U~uD7Jovj7kECqI9Q&+ zY&hA_qzv2?N2p#~{2fS6#~#5vK$wUv%8SaGVc4NlipV?P@Es(r8}eX2DuUBbDi;on z|4Bc5l$pwto~|K03Cj-bH*6tS(O^34wSenj0_^4j@u&b?#2Bdx3KFg>XfNouoTxxp zO>2{id3%s@3wp17+_HH>$3Ce0pz4E-9sIsTr-1IEx3wW4&s{t4U`^k1)&r6*Dofc$ z@942AhHn;lgRl#IpY$OCB}R|dFt~V<|Knh31`jL5VUiW-2FaJ2<}0S9?o%Mu5ooAH zN_)ZZ{Crgq?!6-gnFh&WL?@?I)=ibw^Cxq(L^&`RQ+&o(q=u<$2ofe7QhWpc8;{dZ zc6{-pnUkh~o`pAS-#eG+08sC$V^mDE4O0RaYH|XffdBZQok%(xu0)5rVAS!J>jC}9 z71UGF$C+)W?*H ziV=&A)JB>%16(5=TsHg|F`i;P=k$9T8X~SMf(d;j`(GDoANuTY7ReGwY?YXMO40>B zZ)%A&LMH(eY-9&@#sY3lou)z{gq4)pUkqB`A0HfSF0{tKpqH{|f)2OU3bhWa>DL0Hv1PX7tE$p>n>EQ$IjNJd; zf*t-nE%S%lW*{3b*Au;||GVW!-;iYn{LQ<#Am1!sEv-ro()#Dj#3(Va(X09$?ZLu2 zq((rsl^XO!gGPP!qpwFD{oSu!S*_rTT1^j7U!yD*hkT_4qiS#8Mv14ImzGEhH*L*j6eUF;jJf+ zbmJz(ElnT+>@3E9hO$VB7}nLjw>T^W2$O$RUYK>ZSen#v|v->yCux{dN3(vVVOB$_oxe{TGZ zm5N20Nk0reM1Dd~ASE1$to~LMI9y1#llZpK%nRbp0vy7$PI89FlcapOYznQ&ed6P* zTV`%_c^h_Tiw@Qhke)9nHEv_CHBOk&vl=w`+PBFMyhe zF`zpYOR_j@U3AZcd3kdavX#j%@tr3nCwE?Q@V6%=U+@s%peMd4Aa6GS5R9^1kdY&4 zDvPS7QcoW6Wgk$|R55?sl{1t+ZrTBaK>casI*({(@aW*4DRwXuzt|&Tv`Mj8>v-cK z=m>3yJ5bo1&=dHM$6dHkw@{TjZ0ImfQdg1-s_0U~c56awB-cEBBGyq~5+#1>YGR}9 zCfT~&XxI=b&j}_=k!1R=sD&}Oc7!dXKBTj}j0kZVgtbPHYmvzS@co4CiXp>MilJ{E z5BO6t!lD$cpvSo@L%@XE*6Je?3{W15>xhD8X6cJ1Cw|r^s3vn(e2Wp}36B%AtEPRu zDVr#J)8*lzgxx&@MZaIx1Ss{Wk!o4E!nlY4{pj-Qf}q3mRe7c3@%y^MaNwlk=_`nn zEx)2LJRJ6w?OKn7YS00ZxzQMXc1F1RIP!WnQl1R#$U_&<{w>(2+uk34^QNclTnVea zDqFFuE4>I%tDY2Q8vuW=C^9u#VC!4&xZC8z;>Hf_AnvF6xPkW)XVweTAeX(l^B`q7 zc=aj_;|UeVMV*GEluq4Zh$q*ZhRu`3YybL`LZ9g_g+ZTn;-h;xx&FZfjdec%MVg5( zm%waBD)7@VT~@POI#bWOuKRXP|0I#Y+c_O?M2Eb^2842qCEe8ILQ(6qeV@a+Hg=oj zSA)Yv`)7ThS`DUV+E>QK75|3+@-$WZmSBCp+yQ=R*$Qk;B&xk#)J&xvwy%D-xSBBk zWrop#jhm!PX9k48V%lHH(srypEzmm#q-Zn~Wj#D1(wFY-WM>CtUg(sM7|G%2)jHF? zY1a%Zm{Q6SM+Na;Vta5^8BVk}qJFj}n~XlQ0Ekt-WPbqmP`u947SH~kRRzasFzJ+T zwe-n})9Sjfde1eTjC);oc2TX3*+=C)XZUu`i_K;EX9YHhePvir_(FSTss!%T`o^yG z68vnaOemYnn-=ycYfXD8-+{E9h!(~DLlI3(GJAlZEkV6~p=ChPM6GHCpXK)r8Um3$~b6Cq#Kup&E`e z#qz*#)F^mc1uWDq@9@ynu@JlaGi*ub*+EsBPEfWF=K6Va{_xduwU!50_dlZteS9S* z8f>`)fvMNjE>Qj6y(DD$NGKiscjU_?0fmQ~n2aR&GIl2H-bh$VV=8jI7GR0gfTs}# zD`){Ld<9wE0@}O!a{*)|M8>kSOJ8?=&&)o#C27uG%HR`ne>5>~ebP04F8#Y@$ers{ zSI@4&F7z*1rZgy-K8A4dfrKqN;1A2;0iAA-ExJSXV0fg`ksODAe$uNG1EsY=1fL;n zq&i4k4&)F_<`aPrfIwYhaHilXcgBh-fpP_-+4q35Mc9wTYXmr@FVXwDG zI881JGCOTzPh$`LSnhrOJzSyQ-3<;2{dstn-H&T%V+U(Tt$ly6(MuIEH>;Bpb@|S` zB?m@P7V4CX^@-ktyAvOMIk)r)D z72aynk&@I>Ved zTaY`Z9kNN0+i+uX1>LI`RFc@X-?OEyA0j_tF&()I$w+S`?>U}`dWZy&d|Vu_T*)ol zWTO2+hls}V`RBl<&7UYRbxV+oS}3XasTuX{=b;nwl&_TTsU4Ys<60f)>bQq7Ikkr= zp123fCmAt%c@Fdm!gZ9*CGozNNyH7qku!-jx3HOc=zuI5cD+Fvzs+-4HywhyM+Hl7 zplh|4dMo=M*6%`FSuLRjp#q+TG>Te95I2bQg_t(8-V5~>HJ|vK+$W+0-D|yWeUnvc zZ^fYr!Z&o5-`mS8nYIlO1I*S{T7#XOeZfoDw8e@g%Tv0U>2B`_?IMp;1*E(ErSGV; zALNg(OGU*e3RXv<_usD6JJMrBvx}t|4dyJE4;s5?@Kt}CU+b}oZ7<$|(UVd8#grZ% z?tg8r5ED)79w66GG#mv8Bag)mEki84TZ<$=@xB)rh>b6i680pIZP^eWj8W>l`N#uL zD4mUZJc|uXlqq7VQF;lPwR}MI2wlf3qoz z2LF!GSepC+SP33BaVp=)uqo08t?e1$r<^Hb&b~aHUwHAJ(|@ z8^!DFMnNWrpqDAjPWZq>O1+o7du_3 zjqY~G`I9=wuQ#Xrl6nX7j$9hm{P3R@k8M}-{WRK-%qzDiY>+e$6%`)0*{2PFhUUR_ zO;}=hh4#>vp<l2l1F|Yr`eO zA%!`P`?@co?B1o(U1wA*xx^96WF&qSN_Yjm(fDqm6dHdQOT|#*bD3(Dewq!qz3JR&jYBivQhV zHi}^n)~pu&?A;CG!EhdR!8;GD#Smty%wgnpqwGim07r`kUO)Z=3;{fPD%kNlT*)(gU{9Ep#hPq=n+e?* z3Z}is2Dca8+qtkC-U~^6|3KrPk}WvpmFYiPkSO1ZDg%ezNFJf2we(sHzGrnZasA1o z3rUQdg{yJJ0^72bwKk;a71lUx3)*(2!x;QM5uc;=3Evpq34|=KjLVmyox|pu&28AD z{-zCW^`X}^&Be}b+K!Q}U}uNoXE}f}et@s}04FYstCaK{}># z=sxDGIZ%R@U}?0fQ^2{P8@x!4XQZsIgrm@%+c54*uC&|b`*=B!kB8rQfvPYd=YKxb zmAGHU>&a2?lUaIHGuDV{s5!(8cvlvDZ4}hX8;aBpf*zv@veAn^# zbGR9x<~MVjQ2t4=MjX7@M0!~y;x_rknEu6$12O7@c_N;P%Xx^7-moWu$1#o`rC#Gw z2QPwOg&b9>HyNtKfg|J_30huXH~$+iA0Ib%{d|Gt?=wI^Q7o52X=wePXq^`W9TfvU zjqZQQJW47V0%d7C+aS_hbb8nusWk3E2tbSbOxbo7-~qH|PLU#8O9bt}Hm9r+TO%dC zEOiEwab=7aP*dOdaD}8NzyML_8EPx%7^pfy1`td=v6ykW?A6YEE^fHUjPqQ0ZAfH|Z|x6R?R;M7dT_#{?lM2nzSkr~Q#1_I2y@S-kkg1M% zMBN~>8=_2&vitd({%0{FdZljDC{#9et0;CuK{?chfb60-+zpgmdU)OWm~IJiBB@D- z{K$S3?b59zpGaf|eVBy>?axZ8bwOhohl}7u;AdB_O)f%50}+u#csGE$v0p{u#lA+p zmZ5>!Wh5Snt##?o>3@&TKS?!qUnR*$ba!MXtbw9dp(gMpBWdD%_a3#uYy?`OXCNId zD>O{w)hHuytB$mh%w`GOLdb~&x_14ut$V1>o|_ADg6o=Y&BXgBz`zN2$tVYwgE_W; z9JhFKTa6tTFnTJzbU5-9KpXVDAG4rdD13THbeJH*gWma2)Lr|L0%5?k1EAYVyB1p# z>%j3;R}h8d?_)z~3Pb9sQe~jkTtMaGwOkAed?*#;BzOB(T`J`gcfevxRV~R{A$@E@ z2u+Wx^jKoX8w*%QZ)43_6jO^|$fCLU>Ua(pID-2u@B9TSBw(Gy_5}Dgj?)x1c4YKj zLWH#Wz?;OdMp$}`Vlz?e6=!Q%8ul@3MZ3j|NzDoKgpLB-utl`x;2mPHK{oTCG4?)Z zShiUhM8m)!jB|9Fp#&6tTsa$`wPso=7*7x4KJgVnB^Ps`yR+Hyg}46wToD+Ijm`{T zm5_-jbDFYHEuCNr5Nj1qcB1=*9@5meW?Zlx-QsVu*A0bYexryXJ<|JA4$84TC%?yt z$@H142x(I3mEdKAdazZMhHt&T6pmr*@4HLl z6yYvI`*o$4gxMZ9VTEw>$%&LL9)oAfo9rf38sKOLp^{eC8R&Ufx1`3Uf`!yRT4%1CW(b$x6L~Jcs`jR*zMJaI_WpxzA{y ziq0tUlM>!bS@x5qX|RZk}PQS5MTgEm~-Fwi?wj1g!mN7MYh^q|`2xZXBe8I!K3eiq_AD@p41U2Mm|APLn1i<-^G^wN#bCqEM z0kInVzog0PznLtL|9<~b-faM`l-u2B6bwb+A#yBfeQ2wxSW+(!5*Q#^ zFU3pN&ABVPjWsBM7kt9z5%#0$BPmB_w_8$rZxaq!$MmIArQKPeyleT=7XLrr5tYKM zEk}xpj>ZuTUfCxh8Nv^*OpaULh6=&FlQyfic6z%9!#PY}^|YKfxz!R-YRH?crM(_v zPDJz0G9Qim5b2n@kizb%wL;85-fRXj-%t&4WOPNY zmhp|^)JJKuD~yQE4~&2=+Tc-aX#X%fkp3?@ouL*?TQ}K(^ZnVy!93=j0?NOa+n>wR ziHv3yE_`iSqO>f9}V? zxw&|>oxy(}hYy$kwvVR?B+~8Y;_3H#U?IDF++QtLoqU{Jyd1x8S90ecMlWXL1#$=o zx?(B5{PFR7c=-6ZJHYOv?-wC0{oVZnpKyN~{43WTy8N9^f`(WTfK>V+y9p<^vd!@9rg=&ipr` zjx&xV=Q=t%Gmu~c1(k%Qhx>?Oae+Ds3fAwDDd8zHvH=1oKC~<+hy_aeE#_?DkqsQp z@n%ijj@b2c=O47Sk7xQT%(aGeZ1!-Cnd{R=oZ)8Y*0sKli{@=NO&PnWT@()Z2`{Yu zS~zpXeqZo#ociodXL3UdiF2hFYjUf&l$k+vXO1doS5|H*zkrsb>*-f333G6YEKQm)?G4A zkCWJk**`g^suR+$&AKSlM~lxNH`xjv1twt2B&ezQ3JWrknnyawDr}rY;ssta>$qhU zKW9ktc2Va?+(t`jorn4)MqoDceG3@$TwsG~@rOS-wH+qJDd1;$X3YhJ$xH9SpXN4*d=uPnCDB6jWXcr13=WS#A%ny8KaecJ-Hh=mpf|pbywMn=y`d?JgAEm! z6raHBXkRKOTBT8n$P)}(>&Qd-6G=ilw;7=LN2m)3(n#slip28+MusB?I?iF6HM*PM zIla`Ieb#=K{;wmZq28dv6X%fAzm_yLZHNqq;@sLBF9oafIM{}X0jSlKY6%B5e49wKSam>oY>=en zyrPvz4&kPr-lu;EO2_%z4U^C}t}-vy$y*rNlh;TZ6N_=qBGI*9I3E1m3im8^Bw_z`S`ZV6nj*HVUdxHo zO485KhDBSlfJPe-=PP{D4rs*;&LNxQCZKvh5b|I5*iieHc4C7w0|{33s_vc^tkRw&-eaf23YB z!yDR9!a#>KJqsR`#KOPWLn_<}_OE!8LiD^brM#YStF|a)DkR)FFK{ElKG)ChJ0;cY zvrIIW3-WPRNx0>;288b4%Zk^9Zi}&CzOCM|h(0W5ognsGTwOXq>I=jnsxAts4+qCS z*JMX!wj;qD-zKU);MLhh{v?~zwY#V20f=q<_1MD6>h<|^=DUi(A=MW?is3v|flq_A zd~yolp`uXRzT??5R0&kDfMIboIl=t3NPEq48&fL_a@cfBuUi!DwKGK*EE@_3_XLFu zq^tbuGFX2z{Q$?En-c-D4~6aeM|tC8N-x^|*#P;cXZm~*hN1Kbz0O`(;dWhi5$nrC z<9TYvyn(HWM+MuexVuq?ChFpz2d4?=ZX8k&-`*>>u4OGS!L?mue3DR*m4K>B8)0i{ zG|b&R5{>a`pERnSV#h66fP^5yliD9RI z3zj!Lgk~LU`2T><>6HW124@=B>4M0+)K#lfDd#443=1I3Qq!W&kX~P<0*|9@S`G;S zafyN2ZEFaHrts{C4DCs>-fiGcc1E@fguEl+4kCG_6fLt4q7yeMJZh$> z!#WV0I&+vWP=^6MTnPK+?4S=aeGBt)6?y_9GGcte^=mhR>I;Y$<7pDjB#LMv=mj}BA_3&M2ypCKMDQDmutE% zYrt@yoa=^H#aStsx(VRfEgs4F>(dteqxw{fvukr<%=X7P1`Jp(APK8^;HXR7^?p-) z%$Vn4;WII9h>SSlzw3=5Q)~X5t}?P7LDup`YD!`LgAwvBT)csawPbr(9Wi`F3bPSUZYwwD(vXCf4!v?vLNzi&5dbSO`BiZLiBo$Ur%#LE1fvWh*UI#n(_yA zt)6b%20DN(EcP%z=4DJ4POu@)$~{TFdTWikP9PyB2{XX3RyZ2Q`mxWC?2++GfV(;O zM{J1>mKN<6Q&OH%dJJNCgTyPR*o)S`2k_97NXX@#wImk6{>G`F6+W>@6=t~#=%F^* zJek|SEJ%yl<8a+KDprxZ~8uIDQ3C;$3fhgCV+8;bpu;-~p*NkwU z<8jr}VJsNE)((2=7q9*(SIRtmsAHI>NuKU)ec%UeXd3i-m15~jAsg@nd=~U0 zE~N|-9EkF_c_*Wx(G9c!^7o4ov_CfdIIK-v*cyJ zR_s8Ij>>9o8?XG%nYPMzy>_EvW^*(kTjB&=^FRo@gUr3?y!jLevr+^EMN*J%Oj$kMP~y=(#*RK(@hl zptz_8$F^MXV683vxI;waBo*aNNj2aK;bH!N9C3CG3?#@hj5LB(iYHDHANYWyiPeZ2S8KO8 zLI3VlD{?CC&xJwi*rq9eqDC;7F$~mzXdj?8vWue|87$CL#QQ1PRp%a6RB%_*2liRv z(%G3!(Wb>ebhv!yH+l$BA z3RT=OG_7bY=9}bqsnnEp%BLI%omc4!o{rNH=E3+q9qj-Z|d6I0A6oObDlNsEmS8;ELX(=04VBhfS| z!?4LqSxDO_-V|3xfx7lw?s(JPFhY&3DCJXYIQ0sQG}xl#$;|!1Fm-GCi!*ux zv-`;m6^0EYbv!||=``FjVp`03c z!WhU7hQ55H6#`1Ab?0hmO8~bFt+~+bTh**NpY{ zVP)JdZA;{AJd|l!)D1A8y+4?!-*tf+&Ghb{%WT{$@j-_Q5P)MQ-18Z8)(Q-rHP$rm z9(49KWNWW&vz-P%psS5bZ9!0Lw{_RM@vwK|zy-ZpP6Xh5M9_k3Q(k85z{!T9E8kxp zf_9Xlo|IrNjZe*pxK-v+9iLi3dZmsobk7=;<#+U8^wos!_l^{V;KZ%#2pRLbvwu&t z`nfsY{77#5Si5;Y7KpovkOzE7MV(oK-r?>-X0oy~RJN1WZ2MlbZSK}t>Q{ON3YZz=qzeXF4$#8+P}F4XF}zc@2Mz3(SFmm2L1!byrU8p!NIJWBYXLmJTWH@ zEu&W{gxo-F&fMiImY^&d`n?c!1E>ZuA;SlC@Eo32>@OOe{%f^#)k2x!3YKoPm%1N@ z)<-%nQuYK@@wWUP>?jUeZMCxIM^L`JdJ;afrmeDMa?Rt}DQPb6t zr+m5t64syo&O$mvW!T#ukD6X$(6Ng(vQAziqU?1I{&xtnTLx&;0M0FkH>=ajq1tbYZmFJvGAKa?YV^hl27fQ`+**?qUYAV?d;nMOEOrX0*ZY zmA|_?KMr!a6kA$Li4>|=7Aq!Q8%6AsoP{&eWDrQ(KR@xujB_hTl0Xs^!JjMtZ^WNb z|DOm4%gN6`&Muu2O_QM_Y(qu{1qrS|H#bsDGcFMUsA1k464nWiaAgykK*@4=1);nkOBU|}`n;p&ktz5*UCV+y_WEi@ z2i`>bP&P*7TV0jBgQ@Oz+Jx{PwJL=e-BxfwVkT?@%xWEnC0sb2W`oVbDmb#kA{HUa{$bJ(3vNPk=25FLPcc3IYP@(hYAgb5>b7 zZ?CK6<_L_f$@fK1TT@JaimgTPNB6sk8IytM<#YI)z;wQv_KzdDdUak6@E5*x9i_{( z=x#L77|Rn)l9z2~BbTRJeyK<4+Os3*M?)@Phc}HEr>enty9d{amV!7kyy_1F<3O0x zUyjgcjPKV#&~;-y@T@yakA9y4IsNgl^TyV8zBu%oMOh@Skxptt66t-2V`mU!$D;YFQfn28Intc-skv&8bV8D|7DH`u$B?y-vW!V#Q zLO0EoL_`BgIgzTQXKuiX@Ox^?J62Are`8b$W3SV2oX^B! zh!mKkq^zIxUn*4H+{7rkmGCy;G&V}_>P5z6Cx=j%QVlMg;mRq=VYP0eTKFrqIx|z{ z8xIV%Z#R-%9kKq#nARjP>DM*D8cuV#Df4QBCt8t3cFkUtoK#^!CO90*A|H{XfJ~LE za3{L)Hsu~(+Tk0F?L>RC;zb9pN0ML4t(Y}UNuIVq0Z$DLOuz?eZ45uSemEY5i}^Fe zGo26B{F_h4yl4CfvFI4zu3W5B67sKb1hGLW%_NhYDNKwgQu~LH z^9w)NmJ+XeOz+t3_%X_X)7G*$fr*uY317%qrnCmmkcdQeO*SSeNhK6H!0nO=2Q zY1+v(+7{?{RUXMO7Hybwq5DuwySfm+q8F*WD#KB5$g;6_@N_MX@TC1VLErX5lgzxA z$A(<8pMSJcI_;WMwGEBNUI*Xbk^E!Vb-qteyfp)-qSeH&-%XZ66BXgZ*Jk3be%nlA z4F;&C4qPwTl`HLP4?L71=NK-?{liTSxO1sk^9qm6dNIV@<*%A<#z0ej_L`-lb|(i0 zs$VL#8PhvI1{tOuj_ERNWnSeJN6+aR2*799req$ocV!j#5!+rTc9ZIiUmtV7YYBFr z$9T_Pz1zbi^R~PPn`7xFlcAy&U!$+;RSHwFyj1BT>`{Zt$>%u5k6;_rt=CH^+Skzz zAntaA!;d51&M^6B12gZViNQzgKU8V>IvBj}g&3rv*6QHRu+`RqPWEvyxIG9tI0{9u z;ko`X;8|ROh}MJVgVgi13>nuKuItL>*c$M!Yy@|+>KF2Vi3aX}L<7Zy_1+X52&nYG zi3Y^~8ESAgu{5(Y`VVQWYU;$}aG?2}YjhM#2TUnHVv!a#F=mS$sv9nsRM(#7G8vj- zDcFjLsDJHy+afv}K@62uRn?@J8WZGyz2SRrk{<~Z3DL~>P4?E%g6|Wy#%p#6KR9vh za46<`Rt1L14(%sPGW|&^2)q@3uhQ<{EJT0B@7}(<1MhXKp=#mK!Qbv zUSv-~qaGxoj-dBOMNp4{IBQQ-1#z|YwV5?<4rHet0fyJG-t|=DFjhC)exCbbJ`m;0 zc!5_d$$ehx1vGdot}pz&-X~bTTqfCqhNg>FZ@T*=G)JY=Dyn|dpfeK-E6)gP7BLr3 zaZU@HwZwQzHXkjtZx^+|(%?>(F zI<{@6W81cE+sVth_nmvrJMX;TcvYizjs3@*`>Qc(ueHanx#r9Wu9#DLCx(baX~Y}UXR6A}Cz+8awt?8ku>l%5w74x zRypQC_*_89k;KtSIHGBZ@SWWViMDtpjGkje;Z z7(=dzIuSz~&F zdd*^)DVhGgD7L}p3?zUa6)f|?tavC)$Lw?NY0kwkEwp}AeEg0)ND73f+uV|6ZRrs?xs&fdF%N07=FA zpme3cVq^EDYCb(qrKq^Ft6|(Q@x42^FN%en_whb1_MgMTdNFLaG$;33I_PraC-erFCi=V~$d4u&Pj!P4X z>qfBK3nX8<&U4%f}Mw~v~ z?XrTUse#@bK-~#>;CfHS5+obnDxQlM`0E7C*pHUNrqikB-Zm51F8D z7p9goZO)XoajukUOjc@*No8Co(OrrLrI@#TloUou?M#TGxH!zZVxMt+#VlJRrWo-W`wg?+y!du+IaohYTTi9_>a@nzo~-GUfRPS0rD#VN+-{>&TsB+$5zasL<4Z5 zigB-inJ{e)^5|GFv7mrev>|AwX23(12tLm=))y{-Q5Xul)aGMUifmwl)g6%8=!jH!)c4KamDOFr<#u~{2 zSSv+Mgm^|j0*4Gs!|kP*b1Ul3%y8rtUodGRcx$?j%hw6lO%a7YUbhk)8W|nxfQp*v z6CA`!c{iwowOd$M2xd|R&w+(NC;5iu1XeaCF>qFU@y=lDiohh%R(_}6gfVwy;6)CU zVyfSIN90~kB~!8150??4T|Xeu_&5r)-J?oas^Xps1144jjSwXAc+M?GA%E& zxin;}>PF|e+b2|KjZ5Jcu5~d)ZjD{Xrn=ml2EQ2rzT&7vI-LZI# zf2w$b=0o#~`;$R7Aq*TSUUsEb8oX@uCBE#4()iND1DvRzdr9#Gq)cZ@!HSmg*r}ju zEs|9&$-vc|B0m%V_z0&Lp} z^D;hK+mKw0DR6BoN5LB=iyLky4j=JG&*b$+BMp8NeIVS@hsXoj0-<|Q@iWq}<{lSH z(Hw(E$r%Yo5RAI=?g#xq$B)`#OphG&v$0-z-VnN(EhfK8^h_-YiuF{8e#(ltZ;io1 z2Ma4DW-o%xo2Q~KVQ83MoZJ(>A#7%Nz1c&oZ$VB+rRbQh_O|y0JI~IRzdRzaDkiOa z1*q+&Lcph+0TcsV((N4}mw8nc#c!QQ{EU6vdBfM-1VDcHo1lSPMN#Xk;{$x%Z2z@U ze!c$J&@3$j9V6Y>gDplnBXdV5T5}r{TY4*VLt`69V@G;v2_X?#B@y7i8Oi)Xq|nG# zWI+f3AY%mp@OAos6Um4u3kVA+3!L+OnTUwHTt0=-xi?B%@N2}<6EpXh3?*`_R?v9T zE-PUtNr8ltg|b5E0mvhn?#=NQP{~OcKT^^U3-J8tx6i3kOA6cr`jq67tiY#`~Hnb==d|2u==N zfGNh{a>_G>&WIk`gyFu8rq1AM3y?MkzhGmdHod4h^M(Qg{X{{I&YuD=L_YJg;`v1; z^8>N)vfa~@1>yDBs5x3fV)O>~h#>>Tn|a68!8ztYpAq4(?Lxj{H7UFXrk5fG^@AvR zEKfL09n2j_?yX&@GXzZmXHTIS36wi?>^*6aeDfLdjV4@?1nC>6>}`)H9=Hgi3^8Qu zQqm!lEVEVuyI&wJG$TfcST1=cAP-d*9vLYe6N=|3Lu^QiA11w}p4===>Xc{w(?cUU zGJtZ=?C{T@Mqt>n99CePXZv^TPzI@Nk$IT1-vNP}GWNR-{DdO?jT%7wx0l$c3B>&Q zc-l@M-^pF^uJ1F%&NkfpX1Su4HdPN<@D2#wcfu;sC3E4nR+j#04EC86%Y@z4dQywSpQIc z=LtL62Y+)s+-2*feKTqOkg3%&q6Px+D0{F`7^~I0IZh*_s3f~%`813nYpOUu|JeN+ zQAPzLAc-S{3o6k+EhAAEj`M>Mzn=3k6*0<^h3Po2n=+`n*SF1;*8q|{M|Z5iIM0)l z)Lka8>3Kq7u*ou!M5c2zHE-UqfY7vdPe&X+5BEpyI9ykHXS z=JXsK)GmnD2l0okg^?E)&6d{`0UVmWMiMPWy@p_ftkuKmSHSGv3NKC%C*U4T=!ME_ zBJn;B53J4b&7zUIohcHTRNKW#EFvrjw-0CldI%8=(?$6njgAE=fg30SWRp&qS9;lL)jQQv5EN=+2UJpT$rJu|X1e z??-L-yEh~p=?XVmzjr?$B?qs4YZ^w=P^()Y+S3@)Bn5q(o!lRDXA7DPt9NNO-YbVP zRDR=V+)s2wEpY^fhSWPWBqUExG2#btC;s!5T6~1tdEVw;cKLqDG_*7Y&NBZ~l|Tpj zH2h*UJM26h5}Z5aCEzdqgJidF8CVWexg=qja6mlVzf4AP!2JPZ;vjIpf#{pX92af1 z`kr=yXb*{`R^frfK}KYh3#WkRNEglev)?cP0`)OMMjZYUC61nJP7=XH2-~3eMA(>m z?9>&FlRFyUBlWI~A7YV1e=7(<6ZTYudBd5*RcxlYuCo$XLN zWmyy0Govuj989k&$9O8?JAU>>T@00hyTuuHJh)%k6;$rJ*d#Q zsk^6fn+y1{42hgmgWnKgisBRrrdM1@sPyAYMs>m4>!#M!M%K-aykIY@0HXB0l3TF1 zv376$Cs`(t_tfy-ChUcYVov33mrzo3UUakS8r0*vaS%KgzF7sIhleA!LK7nb_STIp zHOY(;9<(hlztyggZ{1HH?mLmr&CesYXvmemNRf9SVxKksPH;bbEih=5dig+X-&I%! zHYAiwsLixrHeN1?U1320^*%82BYLV^LWzh7NxvpRw(1^Z7_L1PzmE_oI|ob~;w_=d zgAJucP|i4xnQ|LEbzk_DeGUl#^&VE4@u!%WW(BQkw!vo_KIPQ@#Ao9dTWENHCKlwP zbey`>JN&I-5(zNqN(B9T?6)lZGMV5ai{xFM-%b|=CD-h)Uhibk&L=xQs7s%>vv6cX zq;YIZ((UkHHlV)W$PNriD18c@SL(6>1l-iJPxmtltI^bL85F`GoWVRfFpDMT9(bvW zUbC@~=PpT4u_Edbm`CuyLsfLV?KuapFnvVTAXyM$sY!~BFsp=76!RMl3cw2=kRcgU z;9R&AxN@0-RALgIj^%v(jAeI>%~;|H<^{W}42;p@{H@U)fSx$Ef^mupbn-QnP0y+eAa$SIv)Mnst5scJa2qNnsDXJaUkzrBA@y+q9 zI+j)*q&7dJeKeW!K}lExTgR|^^sIJE7K#<_EeFyHbOC!t{QBY?Q9n45CnZmaj58-4 zY$*TE&ehVjZ1uBqy{h4|`ivRIWr6_ZbZs!(nmpQv_q`fPzj0S<>qgkIEk^Me4-4`Trt6Rp2zZ(V8 zeg$+O*zqa~7Tu3j+;JF-Zx9nuxVxsKj%m@e1s2tt4Xg=RfEE!8!^zGS+B9wOsw`;m z{XS0Hr-ELFUI(`El(+db!>3FlHk%0JQ><^C zJfj4sK)RnQ*d=?>UN=(aK|e^!5IBzmCEj`N%0xI_53@LU_NC=E3}f37%~Y&kPzuGq zmEyfYOb}BCJmYQcSR$#==7#?##BWl(uF7DZD;*^i!?#~<;GaRuV%Q1|j7S-hnf_bt zKn(FFDcWM|c*9_Cc6fbKGwefwpdRGKg#B*AITRrKiv0&cLBIwO$zjg|lWK_PmL?kV zlIk(rucUeKhtv!^{Q zs%N?^FPGB!O_mafKzReTy11ao4{|4F4;Wz zM|g$}aRF5R2+a{}uK`Ic(@A!Wdz$z~0(`2RV6Jw1i(CaRk^F#bMi%> z5g3wmIYdq7E-DB>5LK$d+0>CG?bmi={i^OR7SvxjViM6+rN}WiZ4Sz3s341QTht?2 z&@{zelU2zUC+8k%pn+xyj`)m^5H4e2u12I6k0~T3DAsA=SLWC#+*wG2?^l7`- z+=|Qf&Dt~sGD!5_bRzBO?;K+>Iye;Ic1|w)7Cb|AfpT}0$^o}pEUPq5RmWz!QV{BL z&fUsBz8E#mvUvlOmi9U(l$MrFYI+n|&6R;>1xm)hk<0715Zi>#+RtploV<4&_o2i* zj(AYFefe{gDGA!?xC^X{x}uK2B7Il~kR)bY5l9-^5;&gh=v`;n|ZmNE=ERgXDM0njvNOm`S>~)#mhtqJAu{d6X6`k#3Ob zM<(;k3Uw?++1|#Z4s+-~(}XlfI+cn1nqTZZ@(|zQCP(W3?cs@GQFoj`;|H|@lonJ} zZmd{ly)>jdAHt~nO@vQH#o0Q@LWbhfQU6!lH@orfz^8s8kF~e^)h3b99He)|B}2Pb zRjQ&wm(vTxy)NNCyrM)*W32`JQQ60XY{=IN1}Q^BOZ5&i_(-o&NL&oE2P8%!pr|H< z;Iw%_uBpKJ(y>WK(Hi?IN#hCr{p_P2ts`>^yplQDbeupLad3SK3R|4q$~hgg!3R-i zM!%UJw)nOA5?CYrIc;B08Xtob{(QqTIRbjNEkSnxKf?ZKs%PBcHS)2h`txEMBW_Ta z8Mv0pM+bvm?gJ#?H`3i`X4zR%H2h>-{TAJH*lro2^YXKUil@k}sh@bf#dD2lEdgaQ zk#1;)FdnulxBmK;7CP1Qwt$To#(;<;-Z%T zK}7N>auz@%oqtmx3X7t4y=7?KhXiFC5zP}Q&QW{LJ=ErRD!cc99(n^$OxQzplbErd z3m_W))O-T2htnS!VBkH{%0M6R3N593*BNFc>|yn?Ntr!iDflvKVO`%Cn%*x&_!d4v zSoyqB;-|k`S*lQ>`!-r@Ghf^y2~ae{c0ftQUDfC=ohz3~G6A?CfA!b$bYQwe z^N5Dy<_RVpJ+$D_auC>fP8Vn2ND$0J6UjN=_#Y<8Gh;A&*oUKv4|)p7l8@iHvf~M_ znIhFAp|2J*jBXR*M>f#s#K1duZ28`gC-2xkI#05YLnvt{Uj!4N|8$I&i@Y0NRi-#87_AL-J`Xal(`}FmgD*zT>pzy)33cWSsf#!k zYH&CsKr?K5>S-9Mh?zRsev!Y#CH0>=l?*|7E|${fOL$P9^jw~qqK2NQ zE=$w{CY)wGuYrqHR#&wyJukjk23OZH9wylF$0$k1T=bXTrLC_WKrql}YA;Y(4mxH) zEZ2jq8>DX3&^7m`I}{ksklc;;7p=Fc98I7AovT6-R1TJrOE7+Ef#KoT@oaspIBMJX zWexH499{PR$?Zu{Obx|p>8suJA}(8CsP>2ySMI?)7s#6%KB)i4UosF0OT(+D0FPLYrt(C0zjSb}3B({o&e(tLK2MJ!5Vz9gsOb-LBg4B+}-l`mmI?+w((^^WG zu_sJC>2**H-w`7`GibZ%#NIy-{s0UszlG5Hc)8p-KAmmPt9t~Wl+*3x0n|7Hz;~Z~ zkj?7k=Hhs31wM=FB7Y68+7jE6yWAV!SdIJ1d=IcZam&tt8wQWhb0I^ji(>6s1k^dI zUHaJ9F*>(EfuY?8BXP%143d4Wb5#W=y?G7}%48W#^l8&6T?1ayD>TooQYEjF75XWm zYuQ#>rK0EqAPyjue)l8VT@v4cj!fbQ0Z4Ogk@rH*R5B?7?nYa`%HrU*wrlZ?0=8!F zrOg_a^Cuj(drk=wmhj4Q)U?`amS&tVSj|JhGoT<}#(+)JB$~4G*PpE~UAA7st@Zdf zm$bKOZ}IMea?Z6DEjjrc$-^(-FtOwF@Ir=oZ6_^Mg6j*LK$?VOo+_3QdvlhM^ms_x zcEMI*p*QM$X7EW+yzQvEs-2>B=$;FS5+#CpkBQX9cEo>lAgw+%5%B;stC2Pi^N(?@ zcaK{j6+h)=9M_Q*@z7)ka}bs1UWd7~YmeB-L`D{@b7ECswu{=i<+m_X$5KR;3Zp8i zOoCXln2Yi-SvB}YtQ3IX8mf;g}mgmdXDs&t{_JlrnHU%b^7M$@ZQY9)*n&=Yp3K4 zvor_{zd!>H6_94)w0(VBYX9M$=-_#%Kd(IV6FVZi!6LyJYjxpX{rF~;Yngd+i4pHC zmGJ_BM06*~sImobwe);&1AV0=OFz#hX30suW?-`)qoRKT&eBAbJ<`S-qRM;8z}9JD zPR6%VMz503&rr^-obP9^?HpzL3TE0!49@#DE&Tq`$Ygs;_A`D+|#e-`ELj9 zIs*8c#mT6h8>;j^MR$x@K+XgRN>#zLV|SI49#zAanaaqkvUyh-4)u$sEuA#9ucRE3 z8Ch?Z8aIn+iOT-Zi)VY#$|Y?UhQsePR9J| zvaoPAWCdvXn&bNOr#&>`U#XtIrZ;7-6P3~}Axu++Pg3ODcL2%!9Iw0&yWNE&Dcc_z z=~3=uON8Z-oyE?0gPU9gj?Ivk_l&2_i zC_7^y)M*+$LzNe#3D>F`7XsLx94~!z53ClUVa=cW@?%YSWI1#tQ|o@8vxwX88us`1 zNM4vj@t*%dSpzOv0+;~=r`O+9uc#5?-IhVLFc9NtP1Pgst!>YlcQM|+>m3q*b#x>M zR%$MJ-cy2MXrHLG;8wUmmkyI#UQ8q094Squ-#>W@p&rHEmSddtID7w_m9y2lvuj;@ z?L@&j4vFDi@}R9{c6--I$s$AGz%qgNd5j|_%7(tMSx0!iLHKxeR&}945wUu`psqr% z&JR1B#l8|>V1zc0-Kl>tbF+gj*Ep-tuI+es5NUC3!6rzO0i&) z$f2ru4g$5U&5tD@m3!0cyW`oYv@~yf_WfCp{KWlzVMzzq8m{;#E61~_b_(Ac5ipPV zmDPm2sr{Lt78rnaR^Gf{(77M3K$Vybff-3QJ>Y770!EVSWDT*obV(+ySo-XJlS7YCT%&U(7UCjUWM+PoVBwRH(ulkeKPi|kQGnruV^!m{*PQc3v%frOTDZMf z@dUP4kxGV6&0&TOQ^b9}h=CR!2T4mi?a{>{rauHULsr%YeQ0VF?bEKMi`VF97F+rC z+a&KtnAIzczoBxG?VVFO-dHHAGj3P&mumh(I$j=|!jVsCMFV(5Pl`v2qW^uC?*m-y zywI$<*du&$vn!6^DrK> zd{X}%NK5SZLEU$n@47L8gg!VXzFQL@AH`(L3Wq-jy71gwSVLZmMTeGh>#stiLtP5a z$=I6_MZ0p`ZiNPo1L+Q+qkTxoGfQp***APOxuhmf-^ z8FiSGQ9y$KnUcSQ(;+W0`0+Bo+K1#JNv)N7f>oMdb&^8O%W32e*2`%n9jl+EC`1s? z_ukvNrESfUM_-Hy`Jfam49kFeJh07W@ur;HcW(8b=sqow!^2#GF4V1~Vo?1VLh9|S zVFn#lStQls7n|BaEp@Eg{0J)ZUTtgwoJMm&Ks$GNnuXtG7ZSsqM7IGw&3bmIw|A`gaVW8^0TW zd~4G&Rj@PBCNJ>1n6InE6e=_8=tS2q*rsXXTr9dJo(H*mOiW8>e(eaX&?m&ryp||a zXvRHnk!Lbg2gk!4SoNv(-X1)aWFek>7wW}>LOT0mc;&PD<`RwTaYJugxcx#a3kyG) z8!A9%#vcGK;bf#sdYIt@nD%WnuNpDW8<~7CmA~kaiW2(!2fe zQUf5zj?wR7Y1R#s3vHRT(F34$@$WV=xeK~-k(UH~L(`8QB2bCBT zE}VbBMONe19LS;(8F$*s0%6YTi;)FSYr?Mr>-%YjJYqXzPf86+<{d3^vWY!3gJC=K zelNeQ^{AFycx;0AThiq>wCS77VeftOa1@-H{H8-7sn&Y{YL7_tm9;I+^;vh5MTl*@ zQz8zE2$U0zvkfH`>aTO!h|QXaUYpCh$#32+pf<#OT0mdBdYktCkP2sq0%*(e7%Z9e5~d zofAtNs6tJpYh=5&gd8P?lx!IF`{=H-WKYsJvDc#08p6R%9>_JFw-RYP?JV~4!Q`IZ zgK{fWEgELJIYNXix_+B|^ZHKy{)&`LYA{)*<5vwDRAzF&bp~@O#LcC&BX+nT>I5%$ z3cE$7WNNd!mk*~?Bhkr(17uGmb$H;P2zVeX`~DNNaH4JpV+zO178>ff)qJd^ooCW3bh+6oxQT1pP;8?i% zM2<`xr}oB`cd)o)IDM!@w~FQ^ccV=0yswo~{eBTNBMg)hlDodZybd%;Trv%?d(}*M zr1_@()bL98qAO_7U?mhLqYQZ_f!X?HQML><^y3ZfMVv8s{|T7evG-fy58aGrT_};t z;G~86Pe*VI^9x)>U6)@OEv>ox7mb$mK9zeN&(R5Ne1xO=SNe`_TM4@cxrfLrq*nL>jc45vrE~26RfhXleg!!KjeU<|8j(_AvH_>uitG$enMGWit9jh<(8KQ?C?T$|A3Veww}I zeO4UnPc%6E?0TEPs9QPNr6?jKrzq?^GaQ@Cj{-X7?G3F#^0O-! z9j0UqE~1tY14%-JAM^gSH8=0fZufw0vXyNohg0l`E7JbgE4xkud@eO|nD+a!$bE(* zXyp`j9Y)~X3gH>|6Qd`)Vk@tNC6%!kk0jv{MB=C5XL@;agX^4gQ^_U;Ya?vYc%$Vy zzF`~3i+@teA!vv4@2iga_7x<1;0g*!_ujId7jDZ1p4OQ zlUN)H_Lvd6D?s|ya24|5oq1x6%={lEKF#et3mVkY=#pcE49tQT28z z^@eV#$G+EJ$xYb86df<^i*_Xw|E3WFBC6tg_Zt z&9%G%TNb)(YOQm_S>sfNwmy&B>QNDb7^nwe}qWP570F|vcKmYUBxLU zDnwE86AVC$FJ%$`ni(EZMW&b3piuA9uq_y#gy3)2(G#HSMw+5jKWI8dZ=y@RgfcC@ zkbktcVET>w)Y4@Kwbp2=laOwN*t){1UfmKA#KWmoS#DkLfna?!gSmhw@Z*5?C@kzK zjQ)YdI%UShcxyb^TeVb?Ogq3%nJ1M@$+)Rnc~m{~Nf{_6n?2HiZ`Kbg=h-33ff1K& zH81dcdwOw?A>M$gABHsnecpV)+_qQS`Xt~nCXxG z0eQJM2HwpjYjJd&X>7JGj7c#bHG@$xPlNkqUVe8^d5E1;dU+{8AQa&LcgWY*E&E?v zzTY3G|10wA@9=*M@%nEt06@Oqmrc$87N9DTJ-!K`)~61 zzp#3|e`5bf4F7lVzezR!0$&UL3I0>K`FHHU3Ge>G>Wcnv?EjAp@1L~)l;QnFTl}AB z{}ko@lkT6*;x9S{v;VNl|7sooa{l&XHUAIm`|q+Z WF9rHXvjKqp+JnA2#jfQa@BSCw(4%z# literal 0 HcmV?d00001