From 31f8e59e7a4ed232e26dcdc43bc2b1a68f573196 Mon Sep 17 00:00:00 2001 From: Sebastian Mendel Date: Wed, 25 Feb 2026 12:03:36 +0100 Subject: [PATCH] perf(update): use single global executor for full cross-category parallelism Replace per-category sequential ThreadPoolExecutor with a single global executor for all regular tools. Previously each category was processed sequentially with its own executor, creating an unnecessary bottleneck where slow tools in one category blocked all subsequent categories. Also simplify guide.sh tool sorting to plain alphabetical within groups, replacing the complex runtime-first/multi-version-descending sort. Changes: - audit.py: single ThreadPoolExecutor for all regular tools - audit.py: add category label [cat] to progress output - audit.py: add grouped summary with per-category status counts - audit.py: handle CONFLICT/UNKNOWN statuses in summary - audit.py: guard against empty regular_tools (max_workers=0) - audit.py: fix KeyboardInterrupt to cancel futures before with-block exit - audit.py: remove redundant ToolCatalog construction in result loop - guide.sh: replace sort_tools_runtime_first with sort_tools_by_name Signed-off-by: Sebastian Mendel --- audit.py | 203 ++++++++++++++++++++++++++--------------------- scripts/guide.sh | 46 ++--------- 2 files changed, 117 insertions(+), 132 deletions(-) diff --git a/audit.py b/audit.py index 0bf98db..cadfd0e 100755 --- a/audit.py +++ b/audit.py @@ -551,107 +551,128 @@ def cmd_update(args: argparse.Namespace) -> int: # Total includes both regular tools and multi-version entries total = len(regular_tools) + len(multi_version_tools) - # Parallel audit with progress tracking, grouped by category + # Parallel audit with progress tracking + # All tools submitted to a single executor for maximum parallelism + # (no per-category sequential bottleneck) results = [] completed = 0 + # ANSI colors for all platforms + GREEN = "\033[32m" + BOLD_GREEN = "\033[1;32m" + YELLOW = "\033[33m" + BLUE = "\033[34m" + RESET = "\033[0m" + try: + if regular_tools: + with ThreadPoolExecutor(max_workers=min(MAX_WORKERS, len(regular_tools))) as executor: + future_to_tool = {executor.submit(audit_tool, tool, None): tool for tool in regular_tools} + + try: + for future in as_completed(future_to_tool): + tool = future_to_tool[future] + try: + result = future.result() + results.append(result) + completed += 1 + + # Progress + inst = result.get("installed", "") + latest = result.get("latest_upstream", "") + status = result.get("status", "") + cat = tool.category or "general" + + # Color the installed version based on status + if status == "UP-TO-DATE": + inst_color = GREEN + latest_color = GREEN + op = "===" + elif status == "OUTDATED": + inst_color = YELLOW + latest_color = BOLD_GREEN + op = "!==" + elif status == "CONFLICT": + inst_color = YELLOW + latest_color = BOLD_GREEN + op = "āš ļø" + else: # NOT INSTALLED, UNKNOWN + inst_color = BLUE + latest_color = BLUE + op = "?" + + inst_display = inst if inst else "n/a" + latest_display = latest if latest else "n/a" + + # Add pinned/skip markers (reuse catalog from outer scope) + markers = [] + if catalog.is_pinned(tool.name): + markers.append("PINNED") + if catalog.should_skip(tool.name, latest): + markers.append("SKIP") + + marker_str = f" [{' '.join(markers)}]" if markers else "" + inst_fmt = f"{inst_color}{inst_display}{RESET}" + latest_fmt = f"{latest_color}{latest_display}{RESET}" + msg = f"# [{completed}/{total}] [{cat}] {tool.name} (installed: {inst_fmt} {op} latest: {latest_fmt}){marker_str}" + + print(msg, file=sys.stderr, flush=True) + + except Exception as e: + completed += 1 + print(f"# [{completed}/{total}] {tool.name} (failed: {e})", file=sys.stderr, flush=True) + + # Add failure entry + results.append({ + "tool": tool.name, + "category": tool.category, + "installed": "", + "installed_method": "", + "installed_version": "", + "installed_path_selected": "", + "classification_reason_selected": "Detection failed", + "latest_upstream": "", + "latest_version": "", + "upstream_method": tool.source_kind, + "status": "UNKNOWN", + "tool_url": tool_homepage_url(tool), + "latest_url": "", + "hint": "", + }) + except KeyboardInterrupt: + executor.shutdown(wait=False, cancel_futures=True) + raise + + # Print grouped summary + print(f"\n# Summary by category:", file=sys.stderr) for category in sorted_cats: cat_tools = categorized[category] icon = CATEGORY_ICON.get(category, "šŸ“¦") desc = CATEGORY_DESC.get(category, category) - print(f"\n# {icon} {desc} ({len(cat_tools)} tools)", file=sys.stderr) - - with ThreadPoolExecutor(max_workers=min(MAX_WORKERS, len(cat_tools))) as executor: - future_to_tool = {executor.submit(audit_tool, tool, None): tool for tool in cat_tools} - - for future in as_completed(future_to_tool): - tool = future_to_tool[future] - try: - result = future.result() - results.append(result) - completed += 1 - - # Progress - inst = result.get("installed", "") - latest = result.get("latest_upstream", "") - status = result.get("status", "") - - # ANSI colors for all platforms - GREEN = "\033[32m" - BOLD_GREEN = "\033[1;32m" - YELLOW = "\033[33m" - BLUE = "\033[34m" - RESET = "\033[0m" - - # Color the installed version based on status - if status == "UP-TO-DATE": - inst_color = GREEN - latest_color = GREEN - op = "===" - elif status == "OUTDATED": - inst_color = YELLOW - latest_color = BOLD_GREEN # Latest is newer, make it bold green - op = "!==" - elif status == "CONFLICT": - inst_color = YELLOW - latest_color = BOLD_GREEN - op = "āš ļø" - else: # NOT INSTALLED, UNKNOWN - inst_color = BLUE # Blue for not-installed - latest_color = BLUE - op = "?" - - inst_display = inst if inst else "n/a" - latest_display = latest if latest else "n/a" - - # Add pinned/skip markers - from cli_audit.catalog import ToolCatalog - catalog = ToolCatalog() - markers = [] - if catalog.is_pinned(tool.name): - markers.append("PINNED") - if catalog.should_skip(tool.name, latest): - markers.append("SKIP") - - marker_str = f" [{' '.join(markers)}]" if markers else "" - inst_fmt = f"{inst_color}{inst_display}{RESET}" - latest_fmt = f"{latest_color}{latest_display}{RESET}" - msg = f"# [{completed}/{total}] {tool.name} (installed: {inst_fmt} {op} latest: {latest_fmt}){marker_str}" - - print(msg, file=sys.stderr, flush=True) - - except Exception as e: - completed += 1 - print(f"# [{completed}/{total}] {tool.name} (failed: {e})", file=sys.stderr, flush=True) - - # Add failure entry - results.append({ - "tool": tool.name, - "category": tool.category, - "installed": "", - "installed_method": "", - "installed_version": "", - "installed_path_selected": "", - "classification_reason_selected": "Detection failed", - "latest_upstream": "", - "latest_version": "", - "upstream_method": tool.source_kind, - "status": "UNKNOWN", - "tool_url": tool_homepage_url(tool), - "latest_url": "", - "hint": "", - }) + # Count statuses for this category + cat_names = {t.name for t in cat_tools} + cat_results = [r for r in results if r.get("tool") in cat_names] + up_to_date = sum(1 for r in cat_results if r.get("status") == "UP-TO-DATE") + outdated = sum(1 for r in cat_results if r.get("status") == "OUTDATED") + not_installed = sum(1 for r in cat_results if r.get("status") == "NOT INSTALLED") + conflict = sum(1 for r in cat_results if r.get("status") == "CONFLICT") + unknown = sum(1 for r in cat_results if r.get("status") == "UNKNOWN") + parts = [] + if up_to_date: + parts.append(f"{GREEN}{up_to_date} current{RESET}") + if outdated: + parts.append(f"{YELLOW}{outdated} outdated{RESET}") + if not_installed: + parts.append(f"{BLUE}{not_installed} missing{RESET}") + if conflict: + parts.append(f"{YELLOW}{conflict} conflict{RESET}") + if unknown: + parts.append(f"{BLUE}{unknown} unknown{RESET}") + summary = ", ".join(parts) if parts else "ā€“" + print(f"# {icon} {desc}: {summary}", file=sys.stderr) # Audit multi-version runtimes if multi_version_tools: - # ANSI colors - GREEN = "\033[32m" - BOLD_GREEN = "\033[1;32m" - YELLOW = "\033[33m" - BLUE = "\033[34m" - RESET = "\033[0m" - print(f"\n# šŸ”„ Multi-version runtimes ({len(multi_version_tools)} runtimes)", file=sys.stderr) for tool_name, (catalog_data, mv_config) in multi_version_tools.items(): @@ -686,8 +707,6 @@ def cmd_update(args: argparse.Namespace) -> int: print(f"# ā†’ {versioned_name}: {inst_fmt} {op} {latest_fmt}", file=sys.stderr, flush=True) except KeyboardInterrupt: - # Shutdown executor immediately without waiting for threads - executor.shutdown(wait=False, cancel_futures=True) print("\n\nāœ— Interrupted", file=sys.stderr) # Reset terminal state before exiting print("\033[0m", end="", file=sys.stderr, flush=True) diff --git a/scripts/guide.sh b/scripts/guide.sh index 431baa5..7fe5a2f 100755 --- a/scripts/guide.sh +++ b/scripts/guide.sh @@ -799,46 +799,12 @@ category_matches_filter() { return 1 } -# Helper: sort tools with runtime first, then multi-version, then others -# Priority: base runtime (0) < multi-version (1-99 by version desc) < other tools (100) -sort_tools_runtime_first() { +# Helper: sort tools alphabetically by name within a category +sort_tools_by_name() { local tools="$1" - local category="$2" - - # Runtimes that should appear first in their category - local runtimes="python node php go ruby rust" - for tool in $tools; do - local sort_key="999" # Default: other tools sort last - - # Check if tool is a base runtime - for rt in $runtimes; do - if [ "$tool" = "$rt" ]; then - sort_key="000" - break - fi - done - - # Check if tool is a multi-version runtime (e.g., python@3.14) - if [ "$sort_key" = "999" ] && [[ "$tool" == *"@"* ]]; then - local base="${tool%%@*}" - local version="${tool##*@}" - for rt in $runtimes; do - if [ "$base" = "$rt" ]; then - # Sort by version descending (higher versions first) - # Convert version to sortable number (3.14 -> 986, 3.13 -> 987) - local major="${version%%.*}" - local minor="${version#*.}" - minor="${minor%%.*}" - # Invert so higher versions sort first - sort_key=$(printf "%03d" $((999 - major * 10 - minor))) - break - fi - done - fi - - echo "$sort_key $tool" - done | sort | awk '{print $2}' + echo "$tool" + done | sort } # Process tools grouped by category (in category order) @@ -846,8 +812,8 @@ for category in $(printf '%s\n' "${!CATEGORY_TOOLS[@]}" | while read c; do echo tools="${CATEGORY_TOOLS[$category]}" [ -z "$tools" ] && continue - # Sort tools: runtime first, then multi-version (desc), then others - tools="$(sort_tools_runtime_first "$tools" "$category")" + # Sort tools alphabetically by name + tools="$(sort_tools_by_name "$tools")" # Skip if category doesn't match filter if ! category_matches_filter "$category"; then