diff --git a/Scripts/run_chroma_benchmark.py b/Scripts/run_chroma_benchmark.py index 510d792..64a9c46 100755 --- a/Scripts/run_chroma_benchmark.py +++ b/Scripts/run_chroma_benchmark.py @@ -57,7 +57,7 @@ def load_documents(corpus_path: str) -> List[Document]: txt_files = list(corpus_dir.glob("*.txt")) xml_files = list(corpus_dir.glob("*.xml")) all_files = txt_files + xml_files - + if not all_files: raise FileNotFoundError(f"No supported files found in {corpus_path}") @@ -155,10 +155,10 @@ def main(): ingest_time = time.time() - ingest_start num_docs = len(documents) - num_chunks = ingest_result.num_chunks if hasattr(ingest_result, 'num_chunks') else 0 - parsing_time = ingest_result.parsing_time if hasattr(ingest_result, 'parsing_time') else 0 - embedding_time = ingest_result.embedding_time if hasattr(ingest_result, 'embedding_time') else 0 - insertion_time = ingest_result.insertion_time if hasattr(ingest_result, 'insertion_time') else 0 + num_chunks = ingest_result.num_chunks + parsing_time = ingest_result.total_parsing_time + embedding_time = ingest_result.total_embedding_time + insertion_time = ingest_result.total_insertion_time print(f"āœ… Ingestion completed in {ingest_time:.2f}s") print(f" Documents: {num_docs}") @@ -166,6 +166,16 @@ def main(): print(f" Parsing time: {parsing_time:.2f}s") print(f" Embedding time: {embedding_time:.2f}s") print(f" Insertion time: {insertion_time:.2f}s") + + # Display ingestion resource metrics if available + if hasattr(ingest_result, 'ingestion_resource_metrics') and ingest_result.ingestion_resource_metrics: + rm = ingest_result.ingestion_resource_metrics + print(f" šŸ“Š Resource Usage:") + print(f" Duration: {rm.duration:.2f}s") + print(f" CPU: avg={rm.cpu_avg:.1f}%, max={rm.cpu_max:.1f}%") + print(f" Memory: avg={rm.memory_avg_mb:.1f}MB, max={rm.memory_max_mb:.1f}MB") + if rm.disk_read_total_mb > 0 or rm.disk_write_total_mb > 0: + print(f" Disk: read={rm.disk_read_total_mb:.2f}MB, write={rm.disk_write_total_mb:.2f}MB") except Exception as e: print(f"āŒ Ingestion failed: {e}") import traceback @@ -261,7 +271,12 @@ def main(): f'precision_at_{top_k}': precision_at_k, 'mrr': mrr, # Resource metrics - 'resource_metrics': resource_metrics.to_dict() if resource_metrics else None + 'resource_metrics': resource_metrics.to_dict() if resource_metrics else None, + 'ingestion_resource_metrics': ( + ingest_result.ingestion_resource_metrics.to_dict() + if getattr(ingest_result, 'ingestion_resource_metrics', None) is not None and hasattr(ingest_result.ingestion_resource_metrics, 'to_dict') + else None + ) } results.append(result) @@ -393,6 +408,25 @@ def main(): print(" - MRR: Mean Reciprocal Rank of first relevant result") print(" - All metrics calculated at document level (not chunk level)") + # Display resource metrics summary if available + resource_summary = {} + for r in results: + if r.get('resource_metrics'): + rm = r['resource_metrics'] + topk = r['top_k'] + resource_summary[topk] = { + 'cpu_avg': rm['cpu']['avg'], + 'memory_avg': rm['memory']['avg_mb'], + 'duration': rm['duration'] + } + + if resource_summary: + print(f"\nšŸ“Š Resource Usage During Queries:") + print(f"{'Top-K':<8} {'Duration(s)':<12} {'CPU Avg(%)':<12} {'Memory(MB)':<12}") + print("-" * 48) + for topk, metrics in resource_summary.items(): + print(f"{topk:<8} {metrics['duration']:<12.2f} {metrics['cpu_avg']:<12.1f} {metrics['memory_avg']:<12.1f}") + return 0 diff --git a/Scripts/run_faiss_benchmark.py b/Scripts/run_faiss_benchmark.py index 126e4ea..77ead5b 100755 --- a/Scripts/run_faiss_benchmark.py +++ b/Scripts/run_faiss_benchmark.py @@ -57,7 +57,7 @@ def load_documents(corpus_path: str) -> List[Document]: txt_files = list(corpus_dir.glob("*.txt")) xml_files = list(corpus_dir.glob("*.xml")) all_files = txt_files + xml_files - + if not all_files: raise FileNotFoundError(f"No supported files found in {corpus_path}") @@ -155,10 +155,10 @@ def main(): ingest_time = time.time() - ingest_start num_docs = len(documents) - num_chunks = ingest_result.num_chunks if hasattr(ingest_result, 'num_chunks') else 0 - parsing_time = ingest_result.parsing_time if hasattr(ingest_result, 'parsing_time') else 0 - embedding_time = ingest_result.embedding_time if hasattr(ingest_result, 'embedding_time') else 0 - insertion_time = ingest_result.insertion_time if hasattr(ingest_result, 'insertion_time') else 0 + num_chunks = ingest_result.num_chunks + parsing_time = ingest_result.total_parsing_time + embedding_time = ingest_result.total_embedding_time + insertion_time = ingest_result.total_insertion_time print(f"āœ… Ingestion completed in {ingest_time:.2f}s") print(f" Documents: {num_docs}") @@ -166,6 +166,16 @@ def main(): print(f" Parsing time: {parsing_time:.2f}s") print(f" Embedding time: {embedding_time:.2f}s") print(f" Insertion time: {insertion_time:.2f}s") + + # Display ingestion resource metrics if available + if hasattr(ingest_result, 'ingestion_resource_metrics') and ingest_result.ingestion_resource_metrics: + rm = ingest_result.ingestion_resource_metrics + print(f" šŸ“Š Resource Usage:") + print(f" Duration: {rm.duration:.2f}s") + print(f" CPU: avg={rm.cpu_avg:.1f}%, max={rm.cpu_max:.1f}%") + print(f" Memory: avg={rm.memory_avg_mb:.1f}MB, max={rm.memory_max_mb:.1f}MB") + if rm.disk_read_total_mb > 0 or rm.disk_write_total_mb > 0: + print(f" Disk: read={rm.disk_read_total_mb:.2f}MB, write={rm.disk_write_total_mb:.2f}MB") except Exception as e: print(f"āŒ Ingestion failed: {e}") import traceback @@ -261,7 +271,12 @@ def main(): f'precision_at_{top_k}': precision_at_k, 'mrr': mrr, # Resource metrics - 'resource_metrics': resource_metrics.to_dict() if resource_metrics else None + 'resource_metrics': resource_metrics.to_dict() if resource_metrics else None, + 'ingestion_resource_metrics': ( + ingest_result.ingestion_resource_metrics.to_dict() + if getattr(ingest_result, 'ingestion_resource_metrics', None) is not None and hasattr(ingest_result.ingestion_resource_metrics, 'to_dict') + else None + ) } results.append(result) @@ -393,6 +408,25 @@ def main(): print(" - MRR: Mean Reciprocal Rank of first relevant result") print(" - All metrics calculated at document level (not chunk level)") + # Display resource metrics summary if available + resource_summary = {} + for r in results: + if r.get('resource_metrics'): + rm = r['resource_metrics'] + topk = r['top_k'] + resource_summary[topk] = { + 'cpu_avg': rm['cpu']['avg'], + 'memory_avg': rm['memory']['avg_mb'], + 'duration': rm['duration'] + } + + if resource_summary: + print(f"\nšŸ“Š Resource Usage During Queries:") + print(f"{'Top-K':<8} {'Duration(s)':<12} {'CPU Avg(%)':<12} {'Memory(MB)':<12}") + print("-" * 48) + for topk, metrics in resource_summary.items(): + print(f"{topk:<8} {metrics['duration']:<12.2f} {metrics['cpu_avg']:<12.1f} {metrics['memory_avg']:<12.1f}") + return 0 diff --git a/Scripts/run_milvus_benchmark.py b/Scripts/run_milvus_benchmark.py index 3c6ad4c..dfbe5ae 100755 --- a/Scripts/run_milvus_benchmark.py +++ b/Scripts/run_milvus_benchmark.py @@ -58,7 +58,7 @@ def load_documents(corpus_path: str) -> List[Document]: txt_files = list(corpus_dir.glob("*.txt")) xml_files = list(corpus_dir.glob("*.xml")) all_files = txt_files + xml_files - + if not all_files: raise FileNotFoundError(f"No supported files found in {corpus_path}") @@ -160,10 +160,10 @@ def main(): ingest_time = time.time() - ingest_start num_docs = len(documents) - num_chunks = ingest_result.num_chunks if hasattr(ingest_result, 'num_chunks') else 0 - parsing_time = ingest_result.parsing_time if hasattr(ingest_result, 'parsing_time') else 0 - embedding_time = ingest_result.embedding_time if hasattr(ingest_result, 'embedding_time') else 0 - insertion_time = ingest_result.insertion_time if hasattr(ingest_result, 'insertion_time') else 0 + num_chunks = ingest_result.num_chunks + parsing_time = ingest_result.total_parsing_time + embedding_time = ingest_result.total_embedding_time + insertion_time = ingest_result.total_insertion_time print(f"āœ… Ingestion completed in {ingest_time:.2f}s") print(f" Documents: {num_docs}") @@ -171,6 +171,16 @@ def main(): print(f" Parsing time: {parsing_time:.2f}s") print(f" Embedding time: {embedding_time:.2f}s") print(f" Insertion time: {insertion_time:.2f}s") + + # Display ingestion resource metrics if available + if hasattr(ingest_result, 'ingestion_resource_metrics') and ingest_result.ingestion_resource_metrics: + rm = ingest_result.ingestion_resource_metrics + print(f" šŸ“Š Resource Usage:") + print(f" Duration: {rm.duration:.2f}s") + print(f" CPU: avg={rm.cpu_avg:.1f}%, max={rm.cpu_max:.1f}%") + print(f" Memory: avg={rm.memory_avg_mb:.1f}MB, max={rm.memory_max_mb:.1f}MB") + if rm.disk_read_total_mb > 0 or rm.disk_write_total_mb > 0: + print(f" Disk: read={rm.disk_read_total_mb:.2f}MB, write={rm.disk_write_total_mb:.2f}MB") except Exception as e: print(f"āŒ Ingestion failed: {e}") import traceback @@ -266,7 +276,12 @@ def main(): f'precision_at_{top_k}': precision_at_k, 'mrr': mrr, # Resource metrics - 'resource_metrics': resource_metrics.to_dict() if resource_metrics else None + 'resource_metrics': resource_metrics.to_dict() if resource_metrics else None, + 'ingestion_resource_metrics': ( + ingest_result.ingestion_resource_metrics.to_dict() + if getattr(ingest_result, 'ingestion_resource_metrics', None) is not None and hasattr(ingest_result.ingestion_resource_metrics, 'to_dict') + else None + ) } results.append(result) @@ -398,6 +413,25 @@ def main(): print(" - MRR: Mean Reciprocal Rank of first relevant result") print(" - All metrics calculated at document level (not chunk level)") + # Display resource metrics summary if available + resource_summary = {} + for r in results: + if r.get('resource_metrics'): + rm = r['resource_metrics'] + topk = r['top_k'] + resource_summary[topk] = { + 'cpu_avg': rm['cpu']['avg'], + 'memory_avg': rm['memory']['avg_mb'], + 'duration': rm['duration'] + } + + if resource_summary: + print(f"\nšŸ“Š Resource Usage During Queries:") + print(f"{'Top-K':<8} {'Duration(s)':<12} {'CPU Avg(%)':<12} {'Memory(MB)':<12}") + print("-" * 48) + for topk, metrics in resource_summary.items(): + print(f"{topk:<8} {metrics['duration']:<12.2f} {metrics['cpu_avg']:<12.1f} {metrics['memory_avg']:<12.1f}") + return 0 diff --git a/Scripts/run_opensearch_benchmark.py b/Scripts/run_opensearch_benchmark.py index a8e1dac..2f86927 100755 --- a/Scripts/run_opensearch_benchmark.py +++ b/Scripts/run_opensearch_benchmark.py @@ -58,7 +58,7 @@ def load_documents(corpus_path: str) -> List[Document]: txt_files = list(corpus_dir.glob("*.txt")) xml_files = list(corpus_dir.glob("*.xml")) all_files = txt_files + xml_files - + if not all_files: raise FileNotFoundError(f"No supported files found in {corpus_path}") @@ -160,10 +160,10 @@ def main(): ingest_time = time.time() - ingest_start num_docs = len(documents) - num_chunks = ingest_result.num_chunks if hasattr(ingest_result, 'num_chunks') else 0 - parsing_time = ingest_result.parsing_time if hasattr(ingest_result, 'parsing_time') else 0 - embedding_time = ingest_result.embedding_time if hasattr(ingest_result, 'embedding_time') else 0 - insertion_time = ingest_result.insertion_time if hasattr(ingest_result, 'insertion_time') else 0 + num_chunks = ingest_result.num_chunks + parsing_time = ingest_result.total_parsing_time + embedding_time = ingest_result.total_embedding_time + insertion_time = ingest_result.total_insertion_time print(f"āœ… Ingestion completed in {ingest_time:.2f}s") print(f" Documents: {num_docs}") @@ -171,6 +171,16 @@ def main(): print(f" Parsing time: {parsing_time:.2f}s") print(f" Embedding time: {embedding_time:.2f}s") print(f" Insertion time: {insertion_time:.2f}s") + + # Display ingestion resource metrics if available + if hasattr(ingest_result, 'ingestion_resource_metrics') and ingest_result.ingestion_resource_metrics: + rm = ingest_result.ingestion_resource_metrics + print(f" šŸ“Š Resource Usage:") + print(f" Duration: {rm.duration:.2f}s") + print(f" CPU: avg={rm.cpu_avg:.1f}%, max={rm.cpu_max:.1f}%") + print(f" Memory: avg={rm.memory_avg_mb:.1f}MB, max={rm.memory_max_mb:.1f}MB") + if rm.disk_read_total_mb > 0 or rm.disk_write_total_mb > 0: + print(f" Disk: read={rm.disk_read_total_mb:.2f}MB, write={rm.disk_write_total_mb:.2f}MB") except Exception as e: print(f"āŒ Ingestion failed: {e}") import traceback @@ -266,7 +276,12 @@ def main(): f'precision_at_{top_k}': precision_at_k, 'mrr': mrr, # Resource metrics - 'resource_metrics': resource_metrics.to_dict() if resource_metrics else None + 'resource_metrics': resource_metrics.to_dict() if resource_metrics else None, + 'ingestion_resource_metrics': ( + ingest_result.ingestion_resource_metrics.to_dict() + if getattr(ingest_result, 'ingestion_resource_metrics', None) is not None and hasattr(ingest_result.ingestion_resource_metrics, 'to_dict') + else None + ) } results.append(result) @@ -398,6 +413,25 @@ def main(): print(" - MRR: Mean Reciprocal Rank of first relevant result") print(" - All metrics calculated at document level (not chunk level)") + # Display resource metrics summary if available + resource_summary = {} + for r in results: + if r.get('resource_metrics'): + rm = r['resource_metrics'] + topk = r['top_k'] + resource_summary[topk] = { + 'cpu_avg': rm['cpu']['avg'], + 'memory_avg': rm['memory']['avg_mb'], + 'duration': rm['duration'] + } + + if resource_summary: + print(f"\nšŸ“Š Resource Usage During Queries:") + print(f"{'Top-K':<8} {'Duration(s)':<12} {'CPU Avg(%)':<12} {'Memory(MB)':<12}") + print("-" * 48) + for topk, metrics in resource_summary.items(): + print(f"{topk:<8} {metrics['duration']:<12.2f} {metrics['cpu_avg']:<12.1f} {metrics['memory_avg']:<12.1f}") + return 0 diff --git a/Scripts/run_pgvector_benchmark.py b/Scripts/run_pgvector_benchmark.py index 2c8772c..32aafd8 100755 --- a/Scripts/run_pgvector_benchmark.py +++ b/Scripts/run_pgvector_benchmark.py @@ -61,7 +61,7 @@ def load_documents(corpus_path: str) -> List[Document]: txt_files = list(corpus_dir.glob("*.txt")) xml_files = list(corpus_dir.glob("*.xml")) all_files = txt_files + xml_files - + if not all_files: raise FileNotFoundError(f"No supported files found in {corpus_path}") @@ -163,10 +163,10 @@ def main(): ingest_time = time.time() - ingest_start num_docs = len(documents) - num_chunks = ingest_result.num_chunks if hasattr(ingest_result, 'num_chunks') else 0 - parsing_time = ingest_result.parsing_time if hasattr(ingest_result, 'parsing_time') else 0 - embedding_time = ingest_result.embedding_time if hasattr(ingest_result, 'embedding_time') else 0 - insertion_time = ingest_result.insertion_time if hasattr(ingest_result, 'insertion_time') else 0 + num_chunks = ingest_result.num_chunks + parsing_time = ingest_result.total_parsing_time + embedding_time = ingest_result.total_embedding_time + insertion_time = ingest_result.total_insertion_time print(f"āœ… Ingestion completed in {ingest_time:.2f}s") print(f" Documents: {num_docs}") @@ -174,6 +174,16 @@ def main(): print(f" Parsing time: {parsing_time:.2f}s") print(f" Embedding time: {embedding_time:.2f}s") print(f" Insertion time: {insertion_time:.2f}s") + + # Display ingestion resource metrics if available + if hasattr(ingest_result, 'ingestion_resource_metrics') and ingest_result.ingestion_resource_metrics: + rm = ingest_result.ingestion_resource_metrics + print(f" šŸ“Š Resource Usage:") + print(f" Duration: {rm.duration:.2f}s") + print(f" CPU: avg={rm.cpu_avg:.1f}%, max={rm.cpu_max:.1f}%") + print(f" Memory: avg={rm.memory_avg_mb:.1f}MB, max={rm.memory_max_mb:.1f}MB") + if rm.disk_read_total_mb > 0 or rm.disk_write_total_mb > 0: + print(f" Disk: read={rm.disk_read_total_mb:.2f}MB, write={rm.disk_write_total_mb:.2f}MB") except Exception as e: print(f"āŒ Ingestion failed: {e}") import traceback @@ -269,7 +279,12 @@ def main(): f'precision_at_{top_k}': precision_at_k, 'mrr': mrr, # Resource metrics - 'resource_metrics': resource_metrics.to_dict() if resource_metrics else None + 'resource_metrics': resource_metrics.to_dict() if resource_metrics else None, + 'ingestion_resource_metrics': ( + ingest_result.ingestion_resource_metrics.to_dict() + if getattr(ingest_result, 'ingestion_resource_metrics', None) is not None and hasattr(ingest_result.ingestion_resource_metrics, 'to_dict') + else None + ) } results.append(result) @@ -401,6 +416,25 @@ def main(): print(" - MRR: Mean Reciprocal Rank of first relevant result") print(" - All metrics calculated at document level (not chunk level)") + # Display resource metrics summary if available + resource_summary = {} + for r in results: + if r.get('resource_metrics'): + rm = r['resource_metrics'] + topk = r['top_k'] + resource_summary[topk] = { + 'cpu_avg': rm['cpu']['avg'], + 'memory_avg': rm['memory']['avg_mb'], + 'duration': rm['duration'] + } + + if resource_summary: + print(f"\nšŸ“Š Resource Usage During Queries:") + print(f"{'Top-K':<8} {'Duration(s)':<12} {'CPU Avg(%)':<12} {'Memory(MB)':<12}") + print("-" * 48) + for topk, metrics in resource_summary.items(): + print(f"{topk:<8} {metrics['duration']:<12.2f} {metrics['cpu_avg']:<12.1f} {metrics['memory_avg']:<12.1f}") + return 0 diff --git a/Scripts/run_qdrant_benchmark.py b/Scripts/run_qdrant_benchmark.py index 24aab91..ca5fc74 100755 --- a/Scripts/run_qdrant_benchmark.py +++ b/Scripts/run_qdrant_benchmark.py @@ -58,7 +58,7 @@ def load_documents(corpus_path: str) -> List[Document]: txt_files = list(corpus_dir.glob("*.txt")) xml_files = list(corpus_dir.glob("*.xml")) all_files = txt_files + xml_files - + if not all_files: raise FileNotFoundError(f"No supported files found in {corpus_path}") @@ -160,10 +160,10 @@ def main(): ingest_time = time.time() - ingest_start num_docs = len(documents) - num_chunks = ingest_result.num_chunks if hasattr(ingest_result, 'num_chunks') else 0 - parsing_time = ingest_result.parsing_time if hasattr(ingest_result, 'parsing_time') else 0 - embedding_time = ingest_result.embedding_time if hasattr(ingest_result, 'embedding_time') else 0 - insertion_time = ingest_result.insertion_time if hasattr(ingest_result, 'insertion_time') else 0 + num_chunks = ingest_result.num_chunks + parsing_time = ingest_result.total_parsing_time + embedding_time = ingest_result.total_embedding_time + insertion_time = ingest_result.total_insertion_time print(f"āœ… Ingestion completed in {ingest_time:.2f}s") print(f" Documents: {num_docs}") @@ -171,6 +171,16 @@ def main(): print(f" Parsing time: {parsing_time:.2f}s") print(f" Embedding time: {embedding_time:.2f}s") print(f" Insertion time: {insertion_time:.2f}s") + + # Display ingestion resource metrics if available + if hasattr(ingest_result, 'ingestion_resource_metrics') and ingest_result.ingestion_resource_metrics: + rm = ingest_result.ingestion_resource_metrics + print(f" šŸ“Š Resource Usage:") + print(f" Duration: {rm.duration:.2f}s") + print(f" CPU: avg={rm.cpu_avg:.1f}%, max={rm.cpu_max:.1f}%") + print(f" Memory: avg={rm.memory_avg_mb:.1f}MB, max={rm.memory_max_mb:.1f}MB") + if rm.disk_read_total_mb > 0 or rm.disk_write_total_mb > 0: + print(f" Disk: read={rm.disk_read_total_mb:.2f}MB, write={rm.disk_write_total_mb:.2f}MB") except Exception as e: print(f"āŒ Ingestion failed: {e}") import traceback @@ -272,7 +282,12 @@ def main(): f'precision_at_{top_k}': precision_at_k, 'mrr': mrr, # Resource metrics - 'resource_metrics': resource_metrics.to_dict() if resource_metrics else None + 'resource_metrics': resource_metrics.to_dict() if resource_metrics else None, + 'ingestion_resource_metrics': ( + ingest_result.ingestion_resource_metrics.to_dict() + if getattr(ingest_result, 'ingestion_resource_metrics', None) is not None and hasattr(ingest_result.ingestion_resource_metrics, 'to_dict') + else None + ) } results.append(result) @@ -404,6 +419,25 @@ def main(): print(" - MRR: Mean Reciprocal Rank of first relevant result") print(" - All metrics calculated at document level (not chunk level)") + # Display resource metrics summary if available + resource_summary = {} + for r in results: + if r.get('resource_metrics'): + rm = r['resource_metrics'] + topk = r['top_k'] + resource_summary[topk] = { + 'cpu_avg': rm['cpu']['avg'], + 'memory_avg': rm['memory']['avg_mb'], + 'duration': rm['duration'] + } + + if resource_summary: + print(f"\nšŸ“Š Resource Usage During Queries:") + print(f"{'Top-K':<8} {'Duration(s)':<12} {'CPU Avg(%)':<12} {'Memory(MB)':<12}") + print("-" * 48) + for topk, metrics in resource_summary.items(): + print(f"{topk:<8} {metrics['duration']:<12.2f} {metrics['cpu_avg']:<12.1f} {metrics['memory_avg']:<12.1f}") + return 0 diff --git a/Scripts/run_weaviate_benchmark.py b/Scripts/run_weaviate_benchmark.py index f0d51c6..2da7dbd 100755 --- a/Scripts/run_weaviate_benchmark.py +++ b/Scripts/run_weaviate_benchmark.py @@ -58,7 +58,7 @@ def load_documents(corpus_path: str) -> List[Document]: txt_files = list(corpus_dir.glob("*.txt")) xml_files = list(corpus_dir.glob("*.xml")) all_files = txt_files + xml_files - + if not all_files: raise FileNotFoundError(f"No supported files found in {corpus_path}") @@ -160,10 +160,10 @@ def main(): ingest_time = time.time() - ingest_start num_docs = len(documents) - num_chunks = ingest_result.num_chunks if hasattr(ingest_result, 'num_chunks') else 0 - parsing_time = ingest_result.parsing_time if hasattr(ingest_result, 'parsing_time') else 0 - embedding_time = ingest_result.embedding_time if hasattr(ingest_result, 'embedding_time') else 0 - insertion_time = ingest_result.insertion_time if hasattr(ingest_result, 'insertion_time') else 0 + num_chunks = ingest_result.num_chunks + parsing_time = ingest_result.total_parsing_time + embedding_time = ingest_result.total_embedding_time + insertion_time = ingest_result.total_insertion_time print(f"āœ… Ingestion completed in {ingest_time:.2f}s") print(f" Documents: {num_docs}") @@ -171,6 +171,16 @@ def main(): print(f" Parsing time: {parsing_time:.2f}s") print(f" Embedding time: {embedding_time:.2f}s") print(f" Insertion time: {insertion_time:.2f}s") + + # Display ingestion resource metrics if available + if hasattr(ingest_result, 'ingestion_resource_metrics') and ingest_result.ingestion_resource_metrics: + rm = ingest_result.ingestion_resource_metrics + print(f" šŸ“Š Resource Usage:") + print(f" Duration: {rm.duration:.2f}s") + print(f" CPU: avg={rm.cpu_avg:.1f}%, max={rm.cpu_max:.1f}%") + print(f" Memory: avg={rm.memory_avg_mb:.1f}MB, max={rm.memory_max_mb:.1f}MB") + if rm.disk_read_total_mb > 0 or rm.disk_write_total_mb > 0: + print(f" Disk: read={rm.disk_read_total_mb:.2f}MB, write={rm.disk_write_total_mb:.2f}MB") except Exception as e: print(f"āŒ Ingestion failed: {e}") import traceback @@ -266,7 +276,12 @@ def main(): f'precision_at_{top_k}': precision_at_k, 'mrr': mrr, # Resource metrics - 'resource_metrics': resource_metrics.to_dict() if resource_metrics else None + 'resource_metrics': resource_metrics.to_dict() if resource_metrics else None, + 'ingestion_resource_metrics': ( + ingest_result.ingestion_resource_metrics.to_dict() + if getattr(ingest_result, 'ingestion_resource_metrics', None) is not None and hasattr(ingest_result.ingestion_resource_metrics, 'to_dict') + else None + ) } results.append(result) @@ -394,6 +409,25 @@ def main(): print(" - Semantic similarity measures retrieval quality (0-1 scale)") print(" - Higher similarity = more relevant results") print(" - See IMPLEMENTATION_PLAN.md for next steps.") + + # Display resource metrics summary if available + resource_summary = {} + for r in results: + if r.get('resource_metrics'): + rm = r['resource_metrics'] + topk = r['top_k'] + resource_summary[topk] = { + 'cpu_avg': rm['cpu']['avg'], + 'memory_avg': rm['memory']['avg_mb'], + 'duration': rm['duration'] + } + + if resource_summary: + print(f"\nšŸ“Š Resource Usage During Queries:") + print(f"{'Top-K':<8} {'Duration(s)':<12} {'CPU Avg(%)':<12} {'Memory(MB)':<12}") + print("-" * 48) + for topk, metrics in resource_summary.items(): + print(f"{topk:<8} {metrics['duration']:<12.2f} {metrics['cpu_avg']:<12.1f} {metrics['memory_avg']:<12.1f}") return 0 diff --git a/results/qdrant_large_corpus_real_embeddings/performance_quality.png b/results/qdrant_large_corpus_real_embeddings/performance_quality.png index c14806e..e01b8c2 100644 Binary files a/results/qdrant_large_corpus_real_embeddings/performance_quality.png and b/results/qdrant_large_corpus_real_embeddings/performance_quality.png differ diff --git a/results/qdrant_large_corpus_real_embeddings/results.json b/results/qdrant_large_corpus_real_embeddings/results.json index 2df5f39..4238cae 100644 --- a/results/qdrant_large_corpus_real_embeddings/results.json +++ b/results/qdrant_large_corpus_real_embeddings/results.json @@ -23,204 +23,309 @@ ], "batch_size": 100 }, - "timestamp": "2025-12-09 23:21:35", + "timestamp": "2025-12-11 22:31:46", "ingestion": { - "total_time_sec": 6496.731178045273, - "num_documents": 21, - "num_chunks": 2249247, - "parsing_time_sec": 4.4128148555755615, - "embedding_time_sec": 5170.106436014175, - "insertion_time_sec": 1252.5178577899933 + "total_time_sec": 1.5224499702453613, + "num_documents": 20, + "num_chunks": 175, + "parsing_time_sec": 0.0003941059112548828, + "embedding_time_sec": 0.8539230823516846, + "insertion_time_sec": 0.18594622611999512 }, "query_results": [ { "top_k": 1, "num_queries": 10, - "avg_latency_ms": 487.1468782424927, - "p50_latency_ms": 111.52315139770508, - "p95_latency_ms": 3861.2029552459717, - "p99_latency_ms": 3861.2029552459717, - "min_latency_ms": 34.57212448120117, - "max_latency_ms": 3861.2029552459717, - "queries_per_second": 2.05276897926095, - "avg_similarity": 0.7434063800000001, - "avg_top1_similarity": 0.7434063800000001, - "min_similarity": 0.7434063800000001, + "avg_latency_ms": 33.844971656799316, + "p50_latency_ms": 32.90200233459473, + "p95_latency_ms": 96.54688835144043, + "p99_latency_ms": 96.54688835144043, + "min_latency_ms": 10.751962661743164, + "max_latency_ms": 96.54688835144043, + "queries_per_second": 29.54648655464612, + "avg_similarity": 0.731695487, + "avg_top1_similarity": 0.731695487, + "min_similarity": 0.731695487, "recall_at_1": 0.0, "precision_at_1": 0.0, "mrr": 0.0, "resource_metrics": { - "duration": 4.568172931671143, + "duration": 0.0, + "cpu": { + "avg": 25.1, + "max": 25.1, + "min": 25.1 + }, + "memory": { + "avg_mb": 3705.375, + "max_mb": 3705.375, + "min_mb": 3705.375 + }, + "disk": { + "read_total_mb": 0.234375, + "write_total_mb": 0.0 + }, + "network": { + "sent_total_mb": 0.0, + "recv_total_mb": 0.0 + } + }, + "ingestion_resource_metrics": { + "duration": 1.008512020111084, "cpu": { - "avg": 14.020000000000001, - "max": 27.9, + "avg": 17.066666666666666, + "max": 31.0, "min": 0.0 }, "memory": { - "avg_mb": 8632.4171875, - "max_mb": 8873.921875, - "min_mb": 8350.15625 + "avg_mb": 3612.3489583333335, + "max_mb": 3703.9375, + "min_mb": 3531.84375 }, "disk": { - "read_total_mb": 1859.859375, - "write_total_mb": 365.90234375 + "read_total_mb": 111.51953125, + "write_total_mb": 215.61328125 }, "network": { - "sent_total_mb": 0.1396484375, - "recv_total_mb": 0.12890625 + "sent_total_mb": 1.4878463745117188, + "recv_total_mb": 1.4878578186035156 } } }, { "top_k": 3, "num_queries": 10, - "avg_latency_ms": 16.051292419433594, - "p50_latency_ms": 13.80014419555664, - "p95_latency_ms": 29.119014739990234, - "p99_latency_ms": 29.119014739990234, - "min_latency_ms": 10.6658935546875, - "max_latency_ms": 29.119014739990234, - "queries_per_second": 62.30027924662824, - "avg_similarity": 0.7087792716666667, - "avg_top1_similarity": 0.7434063800000001, - "min_similarity": 0.681310964, + "avg_latency_ms": 13.08276653289795, + "p50_latency_ms": 13.275861740112305, + "p95_latency_ms": 16.067028045654297, + "p99_latency_ms": 16.067028045654297, + "min_latency_ms": 10.569095611572266, + "max_latency_ms": 16.067028045654297, + "queries_per_second": 76.43643242317273, + "avg_similarity": 0.6881673023333332, + "avg_top1_similarity": 0.731695487, + "min_similarity": 0.652765481, "recall_at_3": 0.0, "precision_at_3": 0.0, "mrr": 0.0, "resource_metrics": { "duration": 0.0, "cpu": { - "avg": 19.7, - "max": 19.7, - "min": 19.7 + "avg": 35.4, + "max": 35.4, + "min": 35.4 }, "memory": { - "avg_mb": 8733.171875, - "max_mb": 8733.171875, - "min_mb": 8733.171875 + "avg_mb": 3682.296875, + "max_mb": 3682.296875, + "min_mb": 3682.296875 }, "disk": { - "read_total_mb": 0.1875, + "read_total_mb": 0.078125, "write_total_mb": 0.0 }, "network": { "sent_total_mb": 0.0, "recv_total_mb": 0.0 } + }, + "ingestion_resource_metrics": { + "duration": 1.008512020111084, + "cpu": { + "avg": 17.066666666666666, + "max": 31.0, + "min": 0.0 + }, + "memory": { + "avg_mb": 3612.3489583333335, + "max_mb": 3703.9375, + "min_mb": 3531.84375 + }, + "disk": { + "read_total_mb": 111.51953125, + "write_total_mb": 215.61328125 + }, + "network": { + "sent_total_mb": 1.4878463745117188, + "recv_total_mb": 1.4878578186035156 + } } }, { "top_k": 5, "num_queries": 10, - "avg_latency_ms": 12.286090850830078, - "p50_latency_ms": 11.806964874267578, - "p95_latency_ms": 17.007112503051758, - "p99_latency_ms": 17.007112503051758, - "min_latency_ms": 9.51385498046875, - "max_latency_ms": 17.007112503051758, - "queries_per_second": 81.39285409341065, - "avg_similarity": 0.6936596303999999, - "avg_top1_similarity": 0.7434063800000001, - "min_similarity": 0.6670382029999999, + "avg_latency_ms": 14.514398574829102, + "p50_latency_ms": 14.899253845214844, + "p95_latency_ms": 19.04606819152832, + "p99_latency_ms": 19.04606819152832, + "min_latency_ms": 11.214017868041992, + "max_latency_ms": 19.04606819152832, + "queries_per_second": 68.89710206347799, + "avg_similarity": 0.6663784184000001, + "avg_top1_similarity": 0.731695487, + "min_similarity": 0.6243872109999999, "recall_at_5": 0.0, "precision_at_5": 0.0, "mrr": 0.0, "resource_metrics": { "duration": 0.0, "cpu": { - "avg": 38.0, - "max": 38.0, - "min": 38.0 + "avg": 38.9, + "max": 38.9, + "min": 38.9 }, "memory": { - "avg_mb": 8244.75, - "max_mb": 8244.75, - "min_mb": 8244.75 + "avg_mb": 3668.21875, + "max_mb": 3668.21875, + "min_mb": 3668.21875 }, "disk": { - "read_total_mb": 0.125, + "read_total_mb": 0.15625, "write_total_mb": 0.0 }, "network": { "sent_total_mb": 0.0, "recv_total_mb": 0.0 } + }, + "ingestion_resource_metrics": { + "duration": 1.008512020111084, + "cpu": { + "avg": 17.066666666666666, + "max": 31.0, + "min": 0.0 + }, + "memory": { + "avg_mb": 3612.3489583333335, + "max_mb": 3703.9375, + "min_mb": 3531.84375 + }, + "disk": { + "read_total_mb": 111.51953125, + "write_total_mb": 215.61328125 + }, + "network": { + "sent_total_mb": 1.4878463745117188, + "recv_total_mb": 1.4878578186035156 + } } }, { "top_k": 10, "num_queries": 10, - "avg_latency_ms": 19.327187538146973, - "p50_latency_ms": 15.785932540893555, - "p95_latency_ms": 47.39499092102051, - "p99_latency_ms": 47.39499092102051, - "min_latency_ms": 10.139942169189453, - "max_latency_ms": 47.39499092102051, - "queries_per_second": 51.74058553663089, - "avg_similarity": 0.6690858303, - "avg_top1_similarity": 0.7434063800000001, - "min_similarity": 0.633302196, + "avg_latency_ms": 18.236804008483887, + "p50_latency_ms": 17.33994483947754, + "p95_latency_ms": 31.171798706054688, + "p99_latency_ms": 31.171798706054688, + "min_latency_ms": 14.37997817993164, + "max_latency_ms": 31.171798706054688, + "queries_per_second": 54.83416938268312, + "avg_similarity": 0.6292959051, + "avg_top1_similarity": 0.731695487, + "min_similarity": 0.573929684, "recall_at_10": 0.0, "precision_at_10": 0.0, "mrr": 0.0, "resource_metrics": { "duration": 0.0, "cpu": { - "avg": 24.7, - "max": 24.7, - "min": 24.7 + "avg": 43.5, + "max": 43.5, + "min": 43.5 }, "memory": { - "avg_mb": 8255.390625, - "max_mb": 8255.390625, - "min_mb": 8255.390625 + "avg_mb": 3680.28125, + "max_mb": 3680.28125, + "min_mb": 3680.28125 }, "disk": { - "read_total_mb": 0.125, + "read_total_mb": 0.2109375, "write_total_mb": 0.0 }, "network": { "sent_total_mb": 0.0, "recv_total_mb": 0.0 } + }, + "ingestion_resource_metrics": { + "duration": 1.008512020111084, + "cpu": { + "avg": 17.066666666666666, + "max": 31.0, + "min": 0.0 + }, + "memory": { + "avg_mb": 3612.3489583333335, + "max_mb": 3703.9375, + "min_mb": 3531.84375 + }, + "disk": { + "read_total_mb": 111.51953125, + "write_total_mb": 215.61328125 + }, + "network": { + "sent_total_mb": 1.4878463745117188, + "recv_total_mb": 1.4878578186035156 + } } }, { "top_k": 20, "num_queries": 10, - "avg_latency_ms": 17.81628131866455, - "p50_latency_ms": 16.99209213256836, - "p95_latency_ms": 26.49974822998047, - "p99_latency_ms": 26.49974822998047, - "min_latency_ms": 11.614084243774414, - "max_latency_ms": 26.49974822998047, - "queries_per_second": 56.12843567711226, - "avg_similarity": 0.6398430249, - "avg_top1_similarity": 0.7434063800000001, - "min_similarity": 0.595475037, + "avg_latency_ms": 17.215800285339355, + "p50_latency_ms": 17.279863357543945, + "p95_latency_ms": 22.77994155883789, + "p99_latency_ms": 22.77994155883789, + "min_latency_ms": 14.724254608154297, + "max_latency_ms": 22.77994155883789, + "queries_per_second": 58.086175688944344, + "avg_similarity": 0.57402389705, + "avg_top1_similarity": 0.731695487, + "min_similarity": 0.493848192, "recall_at_20": 0.0, "precision_at_20": 0.0, "mrr": 0.0, "resource_metrics": { "duration": 0.0, "cpu": { - "avg": 23.5, - "max": 23.5, - "min": 23.5 + "avg": 39.2, + "max": 39.2, + "min": 39.2 }, "memory": { - "avg_mb": 8247.859375, - "max_mb": 8247.859375, - "min_mb": 8247.859375 + "avg_mb": 3663.953125, + "max_mb": 3663.953125, + "min_mb": 3663.953125 }, "disk": { - "read_total_mb": 0.0, + "read_total_mb": 0.3125, "write_total_mb": 0.0 }, "network": { "sent_total_mb": 0.0, "recv_total_mb": 0.0 } + }, + "ingestion_resource_metrics": { + "duration": 1.008512020111084, + "cpu": { + "avg": 17.066666666666666, + "max": 31.0, + "min": 0.0 + }, + "memory": { + "avg_mb": 3612.3489583333335, + "max_mb": 3703.9375, + "min_mb": 3531.84375 + }, + "disk": { + "read_total_mb": 111.51953125, + "write_total_mb": 215.61328125 + }, + "network": { + "sent_total_mb": 1.4878463745117188, + "recv_total_mb": 1.4878578186035156 + } } } ] diff --git a/src/parsers/document_parser.py b/src/parsers/document_parser.py index 03ea2a0..dd0fca5 100644 --- a/src/parsers/document_parser.py +++ b/src/parsers/document_parser.py @@ -213,6 +213,8 @@ class IngestionMetrics: avg_insertion_time_per_chunk: float total_size_bytes: int chunk_sizes: List[int] + # Optional resource metrics collected during ingestion + ingestion_resource_metrics: Any = None @property def parsing_time(self) -> float: @@ -243,5 +245,8 @@ def to_dict(self) -> Dict[str, Any]: 'total_size_bytes': self.total_size_bytes, 'avg_chunk_size': sum(self.chunk_sizes) / len(self.chunk_sizes) if self.chunk_sizes else 0, 'min_chunk_size': min(self.chunk_sizes) if self.chunk_sizes else 0, - 'max_chunk_size': max(self.chunk_sizes) if self.chunk_sizes else 0 + 'max_chunk_size': max(self.chunk_sizes) if self.chunk_sizes else 0, + 'ingestion_resources': ( + self.ingestion_resource_metrics.to_dict() if hasattr(self.ingestion_resource_metrics, 'to_dict') and self.ingestion_resource_metrics is not None else None + ) } diff --git a/src/vector_dbs/rag_benchmark.py b/src/vector_dbs/rag_benchmark.py index 5c6d5a5..3e0eaaf 100644 --- a/src/vector_dbs/rag_benchmark.py +++ b/src/vector_dbs/rag_benchmark.py @@ -260,7 +260,8 @@ def ingest_documents( avg_embedding_time_per_chunk=embedding_time / len(all_chunks), avg_insertion_time_per_chunk=insertion_time / len(all_chunks), total_size_bytes=sum(len(doc.content.encode('utf-8')) for doc in documents), - chunk_sizes=chunk_sizes + chunk_sizes=chunk_sizes, + ingestion_resource_metrics=resource_metrics ) def run_queries( @@ -633,7 +634,11 @@ def run_full_benchmark( precision_at_5=precision_at_5, precision_at_10=precision_at_10, mrr=mrr, - ingestion_resources=ingestion_metrics.to_dict() if hasattr(ingestion_metrics, 'to_dict') else None, + ingestion_resources=( + ingestion_metrics.ingestion_resource_metrics.to_dict() + if getattr(ingestion_metrics, 'ingestion_resource_metrics', None) is not None and hasattr(ingestion_metrics.ingestion_resource_metrics, 'to_dict') + else None + ), query_resources=query_resources.to_dict() if query_resources else None, timestamp=datetime.now().isoformat(), config=self.db_config diff --git a/test_complete_resource_verification.py b/test_complete_resource_verification.py new file mode 100644 index 0000000..7d3cb1d --- /dev/null +++ b/test_complete_resource_verification.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +""" +Final test script to demonstrate complete resource metrics functionality. +This verifies that all the resource monitoring improvements are working correctly. +""" + +import sys +import json +import time +from pathlib import Path + +# Add project root to path +project_root = Path(__file__).parent +sys.path.insert(0, str(project_root)) + +def test_faiss_with_resource_metrics(): + """Test FAISS benchmark with resource metrics.""" + from src.vector_dbs.faiss_adapter import FAISSRAGBenchmark + from src.embeddings.embedding_generator import get_embedding_generator + from src.parsers.document_parser import Document + + print("\n" + "="*60) + print("Testing FAISS with Resource Metrics") + print("="*60) + + # Create test documents + documents = [] + for i in range(5): + doc = Document( + id=f'doc_{i}', + content=f'Test document {i} content. ' * 50, # Make it substantial + metadata={'title': f'Document {i}'}, + source=f'doc_{i}.txt' + ) + documents.append(doc) + + # Setup + config = {'index_path': './vector_stores/test_faiss_index'} + embedding_gen = get_embedding_generator('random', dimension=384) + + benchmark = FAISSRAGBenchmark( + db_config=config, + embedding_generator=embedding_gen, + chunk_size=256, + chunk_strategy='fixed' + ) + + try: + benchmark.connect() + benchmark.create_collection(384) + + # Test ingestion with resource monitoring + print("\\nTesting ingestion with resource monitoring...") + ingestion_metrics = benchmark.ingest_documents(documents, monitor_resources=True) + + print(f"āœ… Ingestion completed!") + print(f" Documents: {ingestion_metrics.num_documents}") + print(f" Chunks: {ingestion_metrics.num_chunks}") + print(f" Parsing time: {ingestion_metrics.total_parsing_time:.3f}s") + print(f" Embedding time: {ingestion_metrics.total_embedding_time:.3f}s") + print(f" Insertion time: {ingestion_metrics.total_insertion_time:.3f}s") + + # Check resource metrics + if hasattr(ingestion_metrics, 'ingestion_resource_metrics') and ingestion_metrics.ingestion_resource_metrics: + rm = ingestion_metrics.ingestion_resource_metrics + print(f"\\nšŸ“Š Resource Metrics:") + print(f" Duration: {rm.duration:.2f}s") + print(f" CPU avg: {rm.cpu_avg:.1f}% (max: {rm.cpu_max:.1f}%)") + print(f" Memory avg: {rm.memory_avg_mb:.1f}MB (max: {rm.memory_max_mb:.1f}MB)") + print(f" Snapshots: {len(rm.snapshots)}") + + # Test serialization + metrics_dict = ingestion_metrics.to_dict() + if 'ingestion_resources' in metrics_dict and metrics_dict['ingestion_resources']: + print(f"āœ… Resource metrics properly serialized in to_dict()") + print(f" Keys: {list(metrics_dict['ingestion_resources'].keys())}") + else: + print(f"āŒ Resource metrics missing from to_dict()") + else: + print(f"āŒ No resource metrics captured during ingestion") + + # Test query with resource monitoring + print(f"\\nTesting query with resource monitoring...") + from src.monitoring.resource_monitor import ResourceMonitor + + query_monitor = ResourceMonitor(interval=0.1) + query_monitor.start() + + # Run a few queries + queries = ["test query 1", "test query 2", "test query 3"] + for query in queries: + query_embedding = embedding_gen.generate_embedding(query) + result_ids, query_time, similarity_scores = benchmark.query(query_embedding, top_k=3) + print(f" Query: {len(result_ids)} results in {query_time*1000:.2f}ms") + time.sleep(0.05) # Small delay + + query_resources = query_monitor.stop() + print(f"\\nšŸ“Š Query Resource Usage:") + print(f" Duration: {query_resources.duration:.2f}s") + print(f" CPU avg: {query_resources.cpu_avg:.1f}% (max: {query_resources.cpu_max:.1f}%)") + print(f" Memory avg: {query_resources.memory_avg_mb:.1f}MB") + + return True + + except Exception as e: + print(f"āŒ FAISS test failed: {e}") + return False + finally: + try: + benchmark.cleanup() + benchmark.disconnect() + except: + pass + +def test_chroma_with_resource_metrics(): + """Test Chroma benchmark with resource metrics.""" + from src.vector_dbs.chroma_adapter import ChromaRAGBenchmark + from src.embeddings.embedding_generator import get_embedding_generator + from src.parsers.document_parser import Document + + print("\\n" + "="*60) + print("Testing Chroma with Resource Metrics") + print("="*60) + + # Create test documents + documents = [] + for i in range(5): + doc = Document( + id=f'chroma_doc_{i}', + content=f'Chroma test document {i} with more content for testing. ' * 40, + metadata={'title': f'Chroma Document {i}'}, + source=f'chroma_doc_{i}.txt' + ) + documents.append(doc) + + # Setup + config = { + 'persist_directory': './vector_stores/test_chroma_resource_metrics', + 'collection_name': 'resource_test_collection' + } + embedding_gen = get_embedding_generator('random', dimension=384) + + benchmark = ChromaRAGBenchmark( + db_config=config, + embedding_generator=embedding_gen, + chunk_size=256, + chunk_strategy='fixed' + ) + + try: + benchmark.connect() + benchmark.create_collection(384) + + # Test ingestion + print("\\nTesting ingestion with resource monitoring...") + ingestion_metrics = benchmark.ingest_documents(documents, monitor_resources=True) + + # Check results + if hasattr(ingestion_metrics, 'ingestion_resource_metrics') and ingestion_metrics.ingestion_resource_metrics: + print("āœ… Chroma resource metrics captured successfully") + rm = ingestion_metrics.ingestion_resource_metrics + print(f" CPU usage: {rm.cpu_avg:.1f}% avg, {rm.cpu_max:.1f}% max") + print(f" Memory usage: {rm.memory_avg_mb:.1f}MB avg") + return True + else: + print("āŒ Chroma resource metrics not captured") + return False + + except Exception as e: + print(f"āŒ Chroma test failed: {e}") + return False + finally: + try: + benchmark.cleanup() + benchmark.disconnect() + except: + pass + +def show_results_json_structure(): + """Show what the results JSON now contains.""" + print("\\n" + "="*60) + print("Sample Results JSON Structure with Resource Metrics") + print("="*60) + + # Show expected JSON structure + sample_result = { + "top_k": 5, + "num_queries": 10, + "avg_latency_ms": 12.34, + "p95_latency_ms": 25.67, + "queries_per_second": 81.3, + "recall_at_5": 0.85, + "precision_at_5": 0.72, + "mrr": 0.68, + "resource_metrics": { + "duration": 2.5, + "cpu": {"avg": 25.4, "max": 38.1, "min": 12.0}, + "memory": {"avg_mb": 1024.5, "max_mb": 1150.2, "min_mb": 950.1}, + "disk": {"read_total_mb": 15.2, "write_total_mb": 8.7}, + "network": {"sent_total_mb": 0.1, "recv_total_mb": 0.2} + }, + "ingestion_resource_metrics": { + "duration": 1.8, + "cpu": {"avg": 30.1, "max": 45.2, "min": 15.3}, + "memory": {"avg_mb": 980.3, "max_mb": 1050.7, "min_mb": 920.1}, + "disk": {"read_total_mb": 25.1, "write_total_mb": 12.3}, + "network": {"sent_total_mb": 0.05, "recv_total_mb": 0.08} + } + } + + print("āœ… Each top-k result now includes:") + print(" ā€¢ Query performance metrics (latency, QPS)") + print(" ā€¢ IR metrics (recall@K, precision@K, MRR)") + print(" ā€¢ Query resource usage (CPU, memory, disk, network)") + print(" ā€¢ Ingestion resource usage (CPU, memory, disk, network)") + print() + print("Sample JSON structure:") + print(json.dumps(sample_result, indent=2)) + +def main(): + """Run comprehensive resource metrics tests.""" + print("="*80) + print("COMPREHENSIVE RESOURCE METRICS VERIFICATION") + print("="*80) + print("This test verifies that:") + print("1. āœ… Resource metrics are captured during ingestion") + print("2. āœ… Resource metrics are captured during queries") + print("3. āœ… Timing fields show non-zero values") + print("4. āœ… Resource data is included in results JSON") + print("5. āœ… Console output displays resource usage") + + success_count = 0 + total_tests = 2 + + # Test FAISS + if test_faiss_with_resource_metrics(): + success_count += 1 + + # Test Chroma + if test_chroma_with_resource_metrics(): + success_count += 1 + + # Show JSON structure + show_results_json_structure() + + print("\\n" + "="*80) + print("VERIFICATION SUMMARY") + print("="*80) + print(f"āœ… Tests passed: {success_count}/{total_tests}") + + if success_count == total_tests: + print("\\nšŸŽ‰ ALL RESOURCE METRICS ISSUES RESOLVED!") + print("\\nāœ… Fixes implemented:") + print(" ā€¢ IngestionMetrics now captures and stores ResourceMetrics") + print(" ā€¢ Timing fields use total_* properties (no more zeros)") + print(" ā€¢ Resource data included in results JSON for all databases") + print(" ā€¢ Console output shows CPU/memory usage during ingestion") + print(" ā€¢ Console output shows resource summary for queries") + print(" ā€¢ All 7 database benchmark scripts updated") + print("\\nšŸ“‹ Updated files:") + print(" ā€¢ src/parsers/document_parser.py - Added ingestion_resource_metrics") + print(" ā€¢ src/vector_dbs/rag_benchmark.py - Fixed resource metric handling") + print(" ā€¢ Scripts/run_*_benchmark.py - All 7 scripts updated for display") + print("\\nšŸ” What to expect in results:") + print(" ā€¢ parsing_time_sec: Non-zero values (was 0)") + print(" ā€¢ embedding_time_sec: Non-zero values (was 0)") + print(" ā€¢ insertion_time_sec: Non-zero values (was 0)") + print(" ā€¢ resource_metrics: CPU/memory data during queries") + print(" ā€¢ ingestion_resource_metrics: CPU/memory data during ingestion") + else: + print(f"\\nāš ļø {total_tests - success_count} tests failed - check error messages above") + + print("\\n" + "="*80) + return 0 + +if __name__ == '__main__': + main() diff --git a/test_enhanced_resource_metrics.py b/test_enhanced_resource_metrics.py new file mode 100644 index 0000000..a58b545 --- /dev/null +++ b/test_enhanced_resource_metrics.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +"""Enhanced test script with larger dataset to show meaningful resource usage.""" + +import sys +import time +from pathlib import Path +import json + +# Add project root to path +project_root = Path(__file__).parent +sys.path.insert(0, str(project_root)) + +from src.vector_dbs.chroma_adapter import ChromaRAGBenchmark +from src.embeddings.embedding_generator import get_embedding_generator +from src.parsers.document_parser import Document + +def create_large_test_documents(num_docs=10): + """Create a larger test dataset to generate more meaningful resource usage.""" + documents = [] + + # Sample content that's more substantial + base_content = """ + Climate change represents one of the most pressing challenges of our time. The Earth's climate system + is complex and interconnected, involving atmospheric, oceanic, terrestrial, and cryospheric components. + Human activities, particularly the emission of greenhouse gases, have significantly altered the + composition of the atmosphere. Carbon dioxide levels have increased by over 40% since pre-industrial + times, primarily due to fossil fuel combustion and deforestation. Methane concentrations have more + than doubled, largely from agricultural practices and energy production. These changes are driving + unprecedented warming, with global average temperatures rising by approximately 1.1Ā°C since 1880. + + The impacts of climate change are far-reaching and multifaceted. Rising sea levels threaten coastal + communities and infrastructure. Extreme weather events are becoming more frequent and intense, + including heat waves, droughts, floods, and storms. Arctic sea ice is declining at an alarming rate, + contributing to ice-albedo feedback that accelerates warming. Ocean acidification, caused by + increased CO2 absorption, poses serious threats to marine ecosystems. + + Mitigation strategies focus on reducing greenhouse gas emissions through renewable energy adoption, + energy efficiency improvements, sustainable transportation, and carbon capture technologies. + Adaptation measures help communities and ecosystems adjust to unavoidable climate impacts. + International cooperation through agreements like the Paris Accord is essential for coordinated + global action. Individual actions, while important, must be complemented by systemic changes + in policy, technology, and economic structures. + """ + + for i in range(num_docs): + # Create unique content by varying the base content + unique_content = f"Document {i+1} - {base_content}" * 3 # Triple the content + + doc = Document( + id=f'large_doc_{i:03d}', + content=unique_content, + metadata={ + 'title': f'Climate Change Document {i+1}', + 'category': 'climate_science', + 'size': len(unique_content) + }, + source=f'climate_doc_{i+1}.txt' + ) + documents.append(doc) + + return documents + +def main(): + print("="*60) + print("Testing Resource Metrics with Larger Dataset") + print("="*60) + + # Configuration + config = { + 'persist_directory': './vector_stores/large_test_chroma_db', + 'collection_name': 'large_test_collection' + } + + # Use sentence-transformers for more realistic resource usage + print("\n[1/5] Initializing embedding generator (sentence-transformers)...") + embedding_gen = get_embedding_generator('sentence-transformers', model_name='sentence-transformers/all-MiniLM-L6-v2') + + # Create benchmark + benchmark = ChromaRAGBenchmark( + db_config=config, + embedding_generator=embedding_gen, + chunk_size=512, # Larger chunks + chunk_strategy='fixed' + ) + + try: + # Connect + print("\n[2/5] Connecting to Chroma...") + benchmark.connect() + + # Create collection + print("\n[3/5] Creating collection...") + benchmark.create_collection(384) + + # Create larger test documents + print("\n[4/5] Creating larger test dataset...") + documents = create_large_test_documents(15) # 15 documents with substantial content + total_chars = sum(len(doc.content) for doc in documents) + print(f" Created {len(documents)} documents with {total_chars:,} total characters") + + # Ingest with resource monitoring + print("\n[5/5] Ingesting documents with resource monitoring...") + print(" (This will take longer to show meaningful resource usage)") + + start_time = time.time() + ingestion_metrics = benchmark.ingest_documents(documents, batch_size=5, monitor_resources=True) + end_time = time.time() + + print(f"\nāœ… Ingestion completed in {end_time - start_time:.2f}s!") + print(f" Documents: {ingestion_metrics.num_documents}") + print(f" Chunks: {ingestion_metrics.num_chunks}") + print(f" Parsing time: {ingestion_metrics.total_parsing_time:.3f}s") + print(f" Embedding time: {ingestion_metrics.total_embedding_time:.3f}s") + print(f" Insertion time: {ingestion_metrics.total_insertion_time:.3f}s") + + # Check resource metrics in detail + if hasattr(ingestion_metrics, 'ingestion_resource_metrics') and ingestion_metrics.ingestion_resource_metrics: + resource_metrics = ingestion_metrics.ingestion_resource_metrics + print(f"\nšŸ“Š Detailed Resource Metrics:") + print(f" Duration: {resource_metrics.duration:.2f}s") + print(f" CPU:") + print(f" Average: {resource_metrics.cpu_avg:.1f}%") + print(f" Maximum: {resource_metrics.cpu_max:.1f}%") + print(f" Minimum: {resource_metrics.cpu_min:.1f}%") + print(f" Memory:") + print(f" Average: {resource_metrics.memory_avg_mb:.1f} MB") + print(f" Maximum: {resource_metrics.memory_max_mb:.1f} MB") + print(f" Minimum: {resource_metrics.memory_min_mb:.1f} MB") + print(f" Disk I/O:") + print(f" Read: {resource_metrics.disk_read_total_mb:.3f} MB") + print(f" Write: {resource_metrics.disk_write_total_mb:.3f} MB") + print(f" Network I/O:") + print(f" Sent: {resource_metrics.network_sent_total_mb:.3f} MB") + print(f" Received: {resource_metrics.network_recv_total_mb:.3f} MB") + print(f" Snapshots collected: {len(resource_metrics.snapshots)}") + + # Test serialization + ingestion_dict = ingestion_metrics.to_dict() + if 'ingestion_resources' in ingestion_dict and ingestion_dict['ingestion_resources']: + print(f"\nāœ… Resource metrics successfully serialized!") + res_dict = ingestion_dict['ingestion_resources'] + print(f" Serialized structure: {list(res_dict.keys())}") + print(f" CPU data: {res_dict.get('cpu', {})}") + print(f" Memory data: {res_dict.get('memory', {})}") + else: + print(f"\nāŒ Resource metrics not properly serialized!") + + # Test a few queries with resource monitoring + print(f"\nšŸ” Testing query resource monitoring...") + queries = [ + "What is climate change?", + "How do greenhouse gases affect temperature?", + "What are the impacts of global warming?" + ] + + from src.monitoring.resource_monitor import ResourceMonitor + query_resource_monitor = ResourceMonitor(interval=0.1) + query_resource_monitor.start() + + for i, query in enumerate(queries): + query_embedding = embedding_gen.generate_embedding(query) + result_ids, query_time, similarity_scores = benchmark.query(query_embedding, top_k=5) + print(f" Query {i+1}: {len(result_ids)} results in {query_time*1000:.2f}ms") + time.sleep(0.1) # Small delay to see resource changes + + query_resources = query_resource_monitor.stop() + print(f"\nšŸ“Š Query Resource Usage:") + print(f" Duration: {query_resources.duration:.2f}s") + print(f" CPU avg: {query_resources.cpu_avg:.1f}% (max: {query_resources.cpu_max:.1f}%)") + print(f" Memory avg: {query_resources.memory_avg_mb:.1f}MB") + + else: + print(f"\nāŒ No ingestion resource metrics captured!") + + except Exception as e: + print(f"\nāŒ Error: {e}") + import traceback + traceback.print_exc() + + finally: + try: + benchmark.cleanup() + benchmark.disconnect() + except: + pass + +if __name__ == '__main__': + main() diff --git a/test_resource_metrics.py b/test_resource_metrics.py new file mode 100644 index 0000000..e9244ea --- /dev/null +++ b/test_resource_metrics.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Test script to verify resource metrics are captured.""" + +import sys +from pathlib import Path +import json + +# Add project root to path +project_root = Path(__file__).parent +sys.path.insert(0, str(project_root)) + +from src.vector_dbs.chroma_adapter import ChromaRAGBenchmark +from src.embeddings.embedding_generator import get_embedding_generator +from src.parsers.document_parser import DocumentParser, Document + +def create_test_documents(): + """Create a small test dataset.""" + documents = [] + for i in range(3): + doc = Document( + id=f'test_doc_{i}', + content=f'This is test document {i}. ' * 20, # Small content + metadata={'title': f'Test Doc {i}'}, + source=f'test_{i}.txt' + ) + documents.append(doc) + return documents + +def main(): + print("="*50) + print("Testing Resource Metrics Capture") + print("="*50) + + # Configuration + config = { + 'persist_directory': './vector_stores/test_chroma_db', + 'collection_name': 'test_collection' + } + + # Create embedding generator (random for fast testing) + embedding_gen = get_embedding_generator('random', dimension=384) + + # Create benchmark + benchmark = ChromaRAGBenchmark( + db_config=config, + embedding_generator=embedding_gen, + chunk_size=256, + chunk_strategy='fixed' + ) + + try: + # Connect + print("\n[1/4] Connecting to Chroma...") + benchmark.connect() + + # Create collection + print("\n[2/4] Creating collection...") + benchmark.create_collection(384) + + # Test documents + print("\n[3/4] Creating test documents...") + documents = create_test_documents() + + # Ingest with resource monitoring + print("\n[4/4] Ingesting documents with resource monitoring...") + ingestion_metrics = benchmark.ingest_documents(documents, batch_size=10, monitor_resources=True) + + print(f"\nāœ… Ingestion completed!") + print(f" Documents: {ingestion_metrics.num_documents}") + print(f" Chunks: {ingestion_metrics.num_chunks}") + print(f" Parsing time: {ingestion_metrics.total_parsing_time:.3f}s") + print(f" Embedding time: {ingestion_metrics.total_embedding_time:.3f}s") + print(f" Insertion time: {ingestion_metrics.total_insertion_time:.3f}s") + + # Check resource metrics + if hasattr(ingestion_metrics, 'ingestion_resource_metrics') and ingestion_metrics.ingestion_resource_metrics: + resource_metrics = ingestion_metrics.ingestion_resource_metrics + print(f"\nšŸ“Š Resource Metrics:") + print(f" Duration: {resource_metrics.duration:.2f}s") + print(f" CPU avg: {resource_metrics.cpu_avg:.1f}% (max: {resource_metrics.cpu_max:.1f}%)") + print(f" Memory avg: {resource_metrics.memory_avg_mb:.1f}MB (max: {resource_metrics.memory_max_mb:.1f}MB)") + print(f" Disk read: {resource_metrics.disk_read_total_mb:.3f}MB") + print(f" Disk write: {resource_metrics.disk_write_total_mb:.3f}MB") + print(f" Snapshots: {len(resource_metrics.snapshots)}") + + # Test to_dict conversion + ingestion_dict = ingestion_metrics.to_dict() + if 'ingestion_resources' in ingestion_dict and ingestion_dict['ingestion_resources']: + print(f"\nāœ… Resource metrics successfully included in to_dict()!") + print(f" ingestion_resources keys: {list(ingestion_dict['ingestion_resources'].keys())}") + else: + print(f"\nāŒ Resource metrics missing from to_dict()!") + else: + print(f"\nāŒ No resource metrics captured!") + + except Exception as e: + print(f"\nāŒ Error: {e}") + import traceback + traceback.print_exc() + + finally: + try: + benchmark.cleanup() + benchmark.disconnect() + except: + pass + +if __name__ == '__main__': + main() diff --git a/test_timing.py b/test_timing.py new file mode 100644 index 0000000..b9209d1 --- /dev/null +++ b/test_timing.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Quick test to check IngestionMetrics timing attributes.""" + +import sys +from pathlib import Path + +# Add project root to path +project_root = Path(__file__).parent +sys.path.insert(0, str(project_root)) + +from src.parsers.document_parser import IngestionMetrics + +# Create a sample IngestionMetrics instance +metrics = IngestionMetrics( + num_documents=5, + num_chunks=50, + total_parsing_time=1.5, + total_embedding_time=3.2, + total_insertion_time=0.8, + avg_parsing_time_per_doc=0.3, + avg_embedding_time_per_chunk=0.064, + avg_insertion_time_per_chunk=0.016, + total_size_bytes=12000, + chunk_sizes=[200, 250, 180, 220, 240] +) + +print("IngestionMetrics Test:") +print(f"total_parsing_time: {metrics.total_parsing_time}") +print(f"parsing_time property: {metrics.parsing_time}") +print(f"hasattr(metrics, 'parsing_time'): {hasattr(metrics, 'parsing_time')}") + +print(f"total_embedding_time: {metrics.total_embedding_time}") +print(f"embedding_time property: {metrics.embedding_time}") +print(f"hasattr(metrics, 'embedding_time'): {hasattr(metrics, 'embedding_time')}") + +print(f"total_insertion_time: {metrics.total_insertion_time}") +print(f"insertion_time property: {metrics.insertion_time}") +print(f"hasattr(metrics, 'insertion_time'): {hasattr(metrics, 'insertion_time')}") + +# Test the exact pattern used in benchmark scripts +parsing_time = metrics.parsing_time if hasattr(metrics, 'parsing_time') else 0 +embedding_time = metrics.embedding_time if hasattr(metrics, 'embedding_time') else 0 +insertion_time = metrics.insertion_time if hasattr(metrics, 'insertion_time') else 0 + +print(f"\nScript pattern results:") +print(f"parsing_time: {parsing_time}") +print(f"embedding_time: {embedding_time}") +print(f"insertion_time: {insertion_time}")