From 249731cada9e56a0a98b27d373708e4198c2d91d Mon Sep 17 00:00:00 2001 From: Mrunmayee Kokitkar Date: Mon, 6 Jul 2026 22:40:20 +0530 Subject: [PATCH 1/3] feat: Implement concurrent file summarization with ThreadPoolExecutor - Replace sequential file summarization with concurrent execution using ThreadPoolExecutor - Limit max workers to 10 to avoid overwhelming API providers - Use thread-safe locks for collecting summaries and errors - Preserve existing error handling and progress bar integration - Significantly reduces processing time for repositories with many files --- repo2readme/cli/main.py | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/repo2readme/cli/main.py b/repo2readme/cli/main.py index 8780274..f5930cb 100644 --- a/repo2readme/cli/main.py +++ b/repo2readme/cli/main.py @@ -5,6 +5,8 @@ import os from repo2readme.utils.tree import generate_tree from repo2readme.utils.detect_language import detect_lang +from concurrent.futures import ThreadPoolExecutor, as_completed +import threading @click.group() @@ -131,22 +133,35 @@ def format_size(size_bytes): summaries = [] errors = [] + summaries_lock = threading.Lock() + errors_lock = threading.Lock() + + def process_document(doc): + meta = doc["metadata"] + try: + lang = detect_lang(meta.get("file_type", "text")) + summary = summarize_file( + file_path=meta["file_path"], + language=lang, + content=doc["content"] + ) + with summaries_lock: + summaries.append(summary) + except Exception as e: + with errors_lock: + errors.append(f"Error processing {meta.get('file_path')}: {e}") + with Progress() as progress: task = progress.add_task("[cyan]Generating summaries...[/cyan]", total=total_documents) - for doc in documents: - meta = doc["metadata"] - try: - lang = detect_lang(meta.get("file_type", "text")) - summary = summarize_file( - file_path=meta["file_path"], - language=lang, - content=doc["content"] - ) - summaries.append(summary) - except Exception as e: - errors.append(f"Error processing {meta.get('file_path')}: {e}") - progress.update(task, advance=1) + # Limit concurrent workers to avoid overwhelming API providers + max_workers = min(10, total_documents) + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = {executor.submit(process_document, doc): doc for doc in documents} + + for future in as_completed(futures): + progress.update(task, advance=1) rprint("[cyan]Generating README...[/cyan]") From 645d63184e17937116762cc75ef721e67c8914d4 Mon Sep 17 00:00:00 2001 From: Mrunmayee Kokitkar Date: Mon, 6 Jul 2026 22:53:34 +0530 Subject: [PATCH 2/3] fix: Handle edge case when total_documents is 0 - Skip ThreadPoolExecutor when there are no documents to summarize - Prevents ValueError when max_workers would be 0 --- repo2readme/cli/main.py | 53 ++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/repo2readme/cli/main.py b/repo2readme/cli/main.py index f5930cb..08e8619 100644 --- a/repo2readme/cli/main.py +++ b/repo2readme/cli/main.py @@ -133,35 +133,38 @@ def format_size(size_bytes): summaries = [] errors = [] - summaries_lock = threading.Lock() - errors_lock = threading.Lock() - def process_document(doc): - meta = doc["metadata"] - try: - lang = detect_lang(meta.get("file_type", "text")) - summary = summarize_file( - file_path=meta["file_path"], - language=lang, - content=doc["content"] - ) - with summaries_lock: - summaries.append(summary) - except Exception as e: - with errors_lock: - errors.append(f"Error processing {meta.get('file_path')}: {e}") - - with Progress() as progress: - task = progress.add_task("[cyan]Generating summaries...[/cyan]", total=total_documents) + # Skip summarization if there are no documents + if total_documents > 0: + summaries_lock = threading.Lock() + errors_lock = threading.Lock() - # Limit concurrent workers to avoid overwhelming API providers - max_workers = min(10, total_documents) + def process_document(doc): + meta = doc["metadata"] + try: + lang = detect_lang(meta.get("file_type", "text")) + summary = summarize_file( + file_path=meta["file_path"], + language=lang, + content=doc["content"] + ) + with summaries_lock: + summaries.append(summary) + except Exception as e: + with errors_lock: + errors.append(f"Error processing {meta.get('file_path')}: {e}") - with ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = {executor.submit(process_document, doc): doc for doc in documents} + with Progress() as progress: + task = progress.add_task("[cyan]Generating summaries...[/cyan]", total=total_documents) + + # Limit concurrent workers to avoid overwhelming API providers + max_workers = min(10, total_documents) - for future in as_completed(futures): - progress.update(task, advance=1) + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = {executor.submit(process_document, doc): doc for doc in documents} + + for future in as_completed(futures): + progress.update(task, advance=1) rprint("[cyan]Generating README...[/cyan]") From a8517b5fa145ecb6dc52c5a9d992522ad53d4e3d Mon Sep 17 00:00:00 2001 From: agsaru Date: Tue, 7 Jul 2026 03:55:43 +0000 Subject: [PATCH 3/3] reduced concurrent workers --- repo2readme/cli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo2readme/cli/main.py b/repo2readme/cli/main.py index 08e8619..1d51535 100644 --- a/repo2readme/cli/main.py +++ b/repo2readme/cli/main.py @@ -158,7 +158,7 @@ def process_document(doc): task = progress.add_task("[cyan]Generating summaries...[/cyan]", total=total_documents) # Limit concurrent workers to avoid overwhelming API providers - max_workers = min(10, total_documents) + max_workers = min(2, total_documents) with ThreadPoolExecutor(max_workers=max_workers) as executor: futures = {executor.submit(process_document, doc): doc for doc in documents}