diff --git a/src/imgtests/reporting/cli.py b/src/imgtests/reporting/cli.py new file mode 100644 index 00000000..a914a42e --- /dev/null +++ b/src/imgtests/reporting/cli.py @@ -0,0 +1,147 @@ +import argparse +import sys +from pathlib import Path +from typing import TYPE_CHECKING, Final + +from imgtests.constant import DISTRIBUTION_DESCRIPTIONS +from imgtests.reporting.distro_comparison_export import ( + add_distro_comparison_arguments, +) +from imgtests.reporting.distro_comparison_status_export import ( + DEFAULT_EPSILON_PERCENT, +) + +if TYPE_CHECKING: + from collections.abc import Sequence + +EXPORT_TABLES: Final = ("experiment", "util_run_result") +DATABASE_COMMAND: Final = "database" +DISTRO_COMPARISON_COMMAND: Final = "distro-comparison" +DISTRO_COMPARISON_STATUS_COMMAND: Final = "comparison-status" + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + raw_args = list(sys.argv[1:] if argv is None else argv) + + parser = argparse.ArgumentParser( + description="Export imgtests database data and report comparisons to XLSX workbooks.", + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + parser = subparsers.add_parser( + DATABASE_COMMAND, + help="export database tables to an XLSX workbook", + ) + add_database_arguments(parser) + parser.set_defaults(command=DATABASE_COMMAND) + + parser = subparsers.add_parser( + DISTRO_COMPARISON_COMMAND, + help="build Poky/SUSE comparison tables and charts from an exported report.xlsx", + ) + add_distro_comparison_arguments(parser) + parser.set_defaults(command=DISTRO_COMPARISON_COMMAND) + + parser = subparsers.add_parser( + DISTRO_COMPARISON_STATUS_COMMAND, + help="build Poky/SUSE PASS/FAIL status tables from an exported report.xlsx", + ) + parser.set_defaults(command=DISTRO_COMPARISON_STATUS_COMMAND) + + return parser.parse_args(raw_args) + + +def add_database_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "output", + type=Path, + help="Path to the output .xlsx file.", + ) + parser.add_argument( + "--db-url", + required=True, + help="SQLAlchemy database URL.", + ) + parser.add_argument( + "--tables", + nargs="+", + choices=EXPORT_TABLES, + default=list(EXPORT_TABLES), + help="Tables to export. Defaults to all project result tables.", + ) + parser.add_argument( + "--configuration-id", + dest="configuration_ids", + action="append", + required=True, + metavar="DISTRO=ID", + type=_parse_configuration_id_override, + help=( + "Configuration id to write for a distribution. " + "Must be passed for each supported distribution, for example: " + "--configuration-id poky=10 --configuration-id suse=20." + ), + ) + + +def add_comparison_report_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "input", + type=Path, + help="Existing report.xlsx generated by the database export.", + ) + parser.add_argument( + "--output", + type=Path, + help=( + "Output .xlsx file or output directory. " + "Default: input filename with _comparison suffix." + ), + ) + parser.add_argument( + "--experiment-ids", + nargs="+", + required=True, + help=( + "Experiment IDs to compare. With two IDs, one Poky/SUSE group is created. " + "With many IDs, comparable Poky/SUSE pairs are built inside the selected IDs." + ), + ) + + +def add_distro_comparison_status_arguments(parser: argparse.ArgumentParser) -> None: + add_comparison_report_arguments(parser) + parser.add_argument( + "--epsilon-percent", + type=float, + default=DEFAULT_EPSILON_PERCENT, + help="Relative tolerance for status decisions, in percent. Default: 1.0.", + ) + + +def _parse_configuration_id_override(value: str) -> tuple[str, int]: + distribution_name, separator, raw_id = value.partition("=") + + if not separator: + msg = "Expected configuration id in DISTRO=ID format." + raise argparse.ArgumentTypeError(msg) + + distribution_name = distribution_name.strip().lower() + raw_id = raw_id.strip() + + if distribution_name not in DISTRIBUTION_DESCRIPTIONS: + valid_names = ", ".join(DISTRIBUTION_DESCRIPTIONS) + msg = f"Unsupported distribution '{distribution_name}'. Available: {valid_names}." + raise argparse.ArgumentTypeError(msg) + + try: + configuration_id = int(raw_id) + except ValueError as exc: + msg = f"Configuration id for '{distribution_name}' must be an integer." + raise argparse.ArgumentTypeError(msg) from exc + + if configuration_id <= 0: + msg = f"Configuration id for '{distribution_name}' must be positive." + raise argparse.ArgumentTypeError(msg) + + return distribution_name, configuration_id diff --git a/src/imgtests/reporting/distro_comparison_common.py b/src/imgtests/reporting/distro_comparison_common.py new file mode 100644 index 00000000..6271ce29 --- /dev/null +++ b/src/imgtests/reporting/distro_comparison_common.py @@ -0,0 +1,786 @@ +from __future__ import annotations + +import logging +import math +import re +from collections import defaultdict +from collections.abc import Callable, Iterator +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Final, Literal, cast, get_args + +from openpyxl.styles import Font, PatternFill + +from imgtests.database.models.experiment import ExperimentType + +if TYPE_CHECKING: + from pathlib import Path + + from openpyxl import Workbook + from openpyxl.worksheet.worksheet import Worksheet + +logger = logging.getLogger(__name__) + +DISTROS: Final = ("poky", "suse") +CONFIGURATION_SHEET: Final = "configuration" +EXPERIMENT_SHEET: Final = "experiment" + +COMPARISON_SELECTION_SHEET: Final = "comparison_selection" +COMPARISON_INDEX_SHEET: Final = "comparison_index" +COMPARISON_DATA_SHEET: Final = "comparison_data" +COMPARISON_CHARTS_PREFIX: Final = "comparison_charts" +CHART_SHEET_PREFIX: Final = "charts" + +NONZERO_EPSILON: Final = 1e-12 +INVALID_NUMERIC_SENTINELS: Final = {-1.0} +SHORT_TOKEN_LENGTH: Final = 3 +EXPLICIT_PAIR_EXPERIMENT_COUNT: Final = 2 +MAX_SHEET_NAME_LENGTH: Final = 31 +ID_COLUMN_SUFFIX: Final = "_id" +EXPERIMENT_TYPE_VALUES: Final = frozenset(get_args(ExperimentType)) + +SKIP_SOURCE_SHEETS: Final = { + COMPARISON_SELECTION_SHEET, + COMPARISON_INDEX_SHEET, + COMPARISON_DATA_SHEET, + COMPARISON_CHARTS_PREFIX, + "comparison_metrics", + "comparison_summary", +} + +LABEL_COLUMNS: Final = { + "configuration_id", + "distribution_description", + "experiment_id", + "run_result_id", + "id", + "test_name", + "tool_name", + "util_type", + "command", + "description", + "started_at", + "ended_at", + "type", + "config_id", + "tests_total", + "tests_passed", + "tests_failed", + "tests_broken", + "tests_skipped", +} + +TECHNICAL_TOKENS: Final = { + "id", + "pid", + "uid", + "gid", + "fd", + "rc", + "code", + "exit", + "status", + "returncode", + "return", + "retval", + "ret", + "socket", + "port", + "host", + "hostname", + "ip", + "addr", + "address", + "local", + "remote", + "timestamp", + "epoch", + "unix", + "version", + "command", + "cmd", + "cookie", + "mss", + "tos", +} + +TIME_PATTERNS: Final = { + "systemd", + "systemd_analyze", + "duration", + "elapsed", + "runtime", + "seconds", + "secs", + "time", + "latency", + "lat", + "clat", + "slat", + "rtt", + "wait", +} + +THROUGHPUT_PATTERNS: Final = { + "bps", + "bits_per_second", + "bitrate", + "bandwidth", + "bw", + "iops", + "ops_per_sec", + "ops_s", + "bogo_ops_s", + "pps", + "requests_per_second", +} + +PERCENT_PATTERNS: Final = { + "percent", + "percentage", + "pct", + "ratio", + "rate", + "util", + "usage", + "cpu_used", +} + +COUNT_PATTERNS: Final = { + "count", + "total", + "num", + "number", + "items", + "requests", + "retransmit", + "failed", + "errors", + "oom", + "broken", + "skipped", +} + +MEMORY_PATTERNS: Final = { + "rss", + "memory", + "mem", + "kb", + "mb", + "bytes", +} + +IPERF_PATTERNS: Final = { + "iperf", + "network_loopback", +} + +STRESS_NG_PATTERNS: Final = { + "stress_ng", +} + +MetricColumnSelector = Callable[[list[str]], list[int]] +MetricNameMapper = Callable[[str, str, str], str] +MetricPredicate = Callable[[str, str, str], bool] + + +@dataclass(frozen=True) +class SheetRows: + name: str + headers: list[str] + rows: list[list[Any]] + + +@dataclass(frozen=True) +class ExperimentInfo: + experiment_id: int + distro: str + configuration_id: int + description: str + experiment_type: ExperimentType | Literal[""] + started_at: str + + +@dataclass(frozen=True) +class ComparisonGroup: + order: int + group_id: str + label: str + short_label: str + experiments: dict[str, ExperimentInfo] + + +@dataclass +class MetricBucket: + logical_test: str + source_sheet: str + metric_name: str + values: dict[str, dict[str, list[float]]] + + +@dataclass(frozen=True) +class MetricWorksheetContext: + sheet_name: str + source_sheet: str + rows: Iterator[tuple[Any, ...]] + headers: list[str] + config_index: int + experiment_index: int + metric_indexes: list[int] + test_name_index: int | None + tool_name_index: int | None + util_type_index: int | None + distribution_index: int | None + + +@dataclass(frozen=True) +class MetricExtractionOptions: + column_selector: MetricColumnSelector + name_mapper: MetricNameMapper | None = None + predicate: MetricPredicate | None = None + require_nonzero: bool = True + + +def is_comparison_sheet(sheet_name: str) -> bool: + return sheet_name in SKIP_SOURCE_SHEETS or sheet_name.startswith( + (f"{COMPARISON_CHARTS_PREFIX}_", f"{CHART_SHEET_PREFIX}_"), + ) + + +def build_configuration_distro_map(workbook: Workbook) -> dict[str, str]: + if CONFIGURATION_SHEET not in workbook.sheetnames: + return {} + ws = workbook[CONFIGURATION_SHEET] + rows = ws.iter_rows(values_only=True) + try: + headers = [normalize_header(value) for value in next(rows)] + except StopIteration: + return {} + + config_index = column_index(headers, "configuration_id") + distro_index = column_index(headers, "distribution_description") + if config_index is None or distro_index is None: + return {} + + result: dict[str, str] = {} + for row in rows: + key = id_key(cell_value(row, config_index)) + distro = normalize_distro(cell_value(row, distro_index)) + if key and distro: + result[key] = distro + return result + + +def build_experiment_info_map( + workbook: Workbook, + config_distro: dict[str, str], +) -> dict[int, ExperimentInfo]: + if EXPERIMENT_SHEET not in workbook.sheetnames: + return {} + ws = workbook[EXPERIMENT_SHEET] + rows = ws.iter_rows(values_only=True) + try: + headers = [normalize_header(value) for value in next(rows)] + except StopIteration: + return {} + + config_index = column_index(headers, "configuration_id") + experiment_index = column_index(headers, "experiment_id") + description_index = column_index(headers, "description") + type_index = column_index(headers, "type") + started_at_index = column_index(headers, "started_at") + if config_index is None or experiment_index is None: + return {} + + result: dict[int, ExperimentInfo] = {} + for row in rows: + experiment_id = int_id_or_none(cell_value(row, experiment_index)) + configuration_id = int_id_or_none(cell_value(row, config_index)) + if experiment_id is None or configuration_id is None: + continue + result[experiment_id] = ExperimentInfo( + experiment_id=experiment_id, + distro=config_distro.get(id_key(configuration_id), ""), + configuration_id=configuration_id, + description=str(cell_value(row, description_index) or "").strip(), + experiment_type=normalize_experiment_type(cell_value(row, type_index)), + started_at=str(cell_value(row, started_at_index) or "").strip(), + ) + return result + + +def build_comparison_groups( + experiment_info: dict[int, ExperimentInfo], + *, + experiment_ids: list[str] | None, +) -> list[ComparisonGroup]: + selected_ids = normalize_selected_ids(experiment_ids) + if experiment_ids and not selected_ids: + logger.warning("No valid integer experiment IDs were provided.") + return [] + + if selected_ids and len(selected_ids) == EXPLICIT_PAIR_EXPERIMENT_COUNT: + ordered_ids = sorted(selected_ids) + infos = [experiment_info.get(experiment_id) for experiment_id in ordered_ids] + if not all(info is not None for info in infos): + missing = [ + str(experiment_id) + for experiment_id, info in zip(ordered_ids, infos, strict=True) + if info is None + ] + logger.warning("Experiment IDs not found in experiment sheet: %s", ", ".join(missing)) + return [] + by_distro = { + info.distro: info for info in infos if info is not None and info.distro in DISTROS + } + if not all(distro in by_distro for distro in DISTROS): + logger.warning( + "Two IDs were provided, but they are not one Poky and one SUSE experiment.", + ) + return [] + return [make_group(by_distro["poky"], by_distro["suse"], order=1)] + + filtered_infos = [ + info + for info in experiment_info.values() + if info.distro in DISTROS and (not selected_ids or info.experiment_id in selected_ids) + ] + grouped: dict[tuple[str, str], dict[str, list[ExperimentInfo]]] = defaultdict( + lambda: defaultdict(list), + ) + for info in filtered_infos: + grouped[(info.description, info.experiment_type)][info.distro].append(info) + + groups: list[ComparisonGroup] = [] + order = 1 + for key in sorted(grouped, key=lambda item: (item[0], item[1])): + by_distro = grouped[key] + if not all(distro in by_distro for distro in DISTROS): + continue + poky_items = sorted(by_distro["poky"], key=experiment_sort_key) + suse_items = sorted(by_distro["suse"], key=experiment_sort_key) + pair_count = min(len(poky_items), len(suse_items)) + if len(poky_items) != len(suse_items): + logger.warning( + "Experiment group '%s/%s' has %s Poky and %s SUSE runs; " + "only %s comparable pairs will be used.", + key[0], + key[1], + len(poky_items), + len(suse_items), + pair_count, + ) + for poky_info, suse_info in zip( + poky_items[:pair_count], + suse_items[:pair_count], + strict=True, + ): + groups.append(make_group(poky_info, suse_info, order=order)) + order += 1 + + return groups + + +def make_group( + poky_info: ExperimentInfo, + suse_info: ExperimentInfo, + *, + order: int, +) -> ComparisonGroup: + group_id = f"exp_{poky_info.experiment_id}_{suse_info.experiment_id}" + label_parts = [part for part in (poky_info.description, poky_info.experiment_type) if part] + base_label = " | ".join(label_parts) or "experiment" + short_label = f"{order}: p{poky_info.experiment_id}/s{suse_info.experiment_id}" + label = ( + f"{base_label} | exp {poky_info.experiment_id} / poky vs " + f"exp {suse_info.experiment_id} / suse" + ) + return ComparisonGroup( + order=order, + group_id=group_id, + label=label, + short_label=short_label, + experiments={"poky": poky_info, "suse": suse_info}, + ) + + +def normalize_selected_ids(experiment_ids: list[str] | None) -> set[int]: + if not experiment_ids: + return set() + selected_ids: set[int] = set() + invalid_ids: list[str] = [] + for experiment_id in experiment_ids: + normalized_id = int_id_or_none(experiment_id) + if normalized_id is None: + invalid_ids.append(str(experiment_id)) + continue + selected_ids.add(normalized_id) + + if invalid_ids: + logger.warning("Experiment IDs must be integers and were ignored: %s", invalid_ids) + return selected_ids + + +def experiment_sort_key(info: ExperimentInfo) -> tuple[str, int]: + return info.started_at, info.experiment_id + + +def build_group_by_experiment(groups: list[ComparisonGroup]) -> dict[str, ComparisonGroup]: + group_by_experiment: dict[str, ComparisonGroup] = {} + for group in groups: + for info in group.experiments.values(): + group_by_experiment[id_key(info.experiment_id)] = group + return group_by_experiment + + +def extract_metric_buckets( + workbook: Workbook, + config_distro: dict[str, str], + groups: list[ComparisonGroup], + options: MetricExtractionOptions, +) -> list[MetricBucket]: + group_by_experiment = build_group_by_experiment(groups) + buckets: dict[tuple[str, str, str], MetricBucket] = {} + + for worksheet in workbook.worksheets: + context = metric_worksheet_context(worksheet, options.column_selector) + if context is None: + continue + for row in context.rows: + collect_metric_buckets_from_row( + row, + context, + group_by_experiment, + config_distro, + buckets, + options, + ) + + return list(buckets.values()) + + +def metric_worksheet_context( + worksheet: Worksheet, + column_selector: MetricColumnSelector, +) -> MetricWorksheetContext | None: + sheet_name = worksheet.title + if is_comparison_sheet(sheet_name): + return None + + rows_iter = worksheet.iter_rows(values_only=True) + try: + headers = [normalize_header(value) for value in next(rows_iter)] + except StopIteration: + return None + + config_index = column_index(headers, "configuration_id") + experiment_index = column_index(headers, "experiment_id") + if config_index is None or experiment_index is None: + return None + + metric_indexes = column_selector(headers) + if not metric_indexes: + return None + + return MetricWorksheetContext( + sheet_name=sheet_name, + source_sheet=strip_excel_suffix(sheet_name), + rows=rows_iter, + headers=headers, + config_index=config_index, + experiment_index=experiment_index, + metric_indexes=metric_indexes, + test_name_index=column_index(headers, "test_name"), + tool_name_index=column_index(headers, "tool_name"), + util_type_index=column_index(headers, "util_type"), + distribution_index=column_index(headers, "distribution_description"), + ) + + +def collect_metric_buckets_from_row( # noqa: PLR0913 + row: tuple[Any, ...], + context: MetricWorksheetContext, + group_by_experiment: dict[str, ComparisonGroup], + config_distro: dict[str, str], + buckets: dict[tuple[str, str, str], MetricBucket], + options: MetricExtractionOptions, +) -> None: + experiment_id = id_key(cell_value(row, context.experiment_index)) + group = group_by_experiment.get(experiment_id) + if group is None: + return + + distro = row_distro(row, context.distribution_index, context.config_index, config_distro) + if distro not in DISTROS: + return + + logical_test = logical_test_name( + context.sheet_name, + cell_value(row, context.test_name_index), + cell_value(row, context.tool_name_index), + cell_value(row, context.util_type_index), + ) + + for metric_index in context.metric_indexes: + metric_name = context.headers[metric_index] + bucket_metric_name = metric_name + if options.name_mapper is not None: + bucket_metric_name = options.name_mapper( + context.source_sheet, + logical_test, + metric_name, + ) + if options.predicate is not None and not options.predicate( + context.source_sheet, + logical_test, + bucket_metric_name, + ): + continue + value = to_finite_float(cell_value(row, metric_index)) + if value is None or is_invalid_metric_value(value): + continue + if options.require_nonzero and not is_nonzero_number(value): + continue + bucket = get_or_create_metric_bucket( + buckets, + logical_test, + context.source_sheet, + bucket_metric_name, + ) + bucket.values[group.group_id][distro].append(value) + + +def get_or_create_metric_bucket( + buckets: dict[tuple[str, str, str], MetricBucket], + logical_test: str, + source_sheet: str, + metric_name: str, +) -> MetricBucket: + bucket_key = (logical_test, source_sheet, metric_name) + bucket = buckets.get(bucket_key) + if bucket is None: + bucket = MetricBucket( + logical_test=logical_test, + source_sheet=source_sheet, + metric_name=metric_name, + values=defaultdict(lambda: defaultdict(list)), + ) + buckets[bucket_key] = bucket + return bucket + + +def find_metric_columns(headers: list[str]) -> list[int]: + metric_indexes: list[int] = [] + for index, header in enumerate(headers): + if not header or header in LABEL_COLUMNS or header.endswith(ID_COLUMN_SUFFIX): + continue + normalized_header = normalize_metric_text(header) + if not normalized_header: + continue + tokens = set(normalized_header.split("_")) + if tokens & TECHNICAL_TOKENS: + continue + metric_indexes.append(index) + return metric_indexes + + +def row_distro( + row: tuple[Any, ...], + distribution_index: int | None, + config_index: int, + config_distro: dict[str, str], +) -> str: + if distribution_index is not None: + distro = normalize_distro(cell_value(row, distribution_index)) + if distro: + return distro + return config_distro.get(id_key(cell_value(row, config_index)), "") + + +def logical_test_name(sheet_name: str, test_name: Any, tool_name: Any, util_type: Any) -> str: + parts: list[str] = [] + prepared_test_name = str(test_name or "").strip() + prepared_tool_name = str(tool_name or "").strip() + prepared_util_type = str(util_type or "").strip() + + if prepared_test_name: + parts.append(prepared_test_name) + else: + parts.append(strip_excel_suffix(sheet_name)) + + if prepared_tool_name and prepared_tool_name.lower() not in prepared_test_name.lower(): + parts.append(prepared_tool_name) + if prepared_util_type: + parts.append(prepared_util_type) + return " / ".join(parts) + + +def strip_excel_suffix(sheet_name: str) -> str: + return re.sub(r"_\d+$", "", str(sheet_name).strip()) + + +def normalize_header(value: Any) -> str: + return "" if value is None else str(value).strip() + + +def column_index(headers: list[str], name: str) -> int | None: + for index, header in enumerate(headers): + if header == name: + return index + return None + + +def cell_value(row: tuple[Any, ...] | list[Any], index: int | None) -> Any: + if index is None or index >= len(row): + return None + return row[index] + + +def row_has_data(row: tuple[Any, ...] | list[Any]) -> bool: + return any(not is_empty_value(value) for value in row) + + +def sheet_has_data_rows(sheet: SheetRows) -> bool: + return any(row_has_data(row) for row in sheet.rows) + + +def is_empty_value(value: Any) -> bool: + return value is None or value == "" + + +def id_key(value: Any) -> str: + if isinstance(value, bool) or value is None: + return "" + if isinstance(value, int): + return str(value) + if isinstance(value, float) and value.is_integer(): + return str(int(value)) + return str(value).strip() + + +def int_id_or_none(value: Any) -> int | None: + if isinstance(value, bool) or value is None: + return None + if isinstance(value, int): + return value + if isinstance(value, float): + return int(value) if value.is_integer() else None + + text = str(value).strip() + if not text: + return None + try: + return int(text) + except ValueError: + return None + + +def normalize_distro(value: Any) -> str: + text = str(value or "").strip().lower() + if "suse" in text: + return "suse" + if "yocto" in text or "poky" in text: + return "poky" + return text + + +def normalize_metric_text(value: str) -> str: + normalized = re.sub(r"[^a-zA-Z0-9]+", "_", str(value).lower()) + normalized = re.sub(r"_+", "_", normalized) + return normalized.strip("_") + + +def normalize_experiment_type(value: Any) -> ExperimentType | Literal[""]: + text = str(value or "").strip() + if text in EXPERIMENT_TYPE_VALUES: + return cast("ExperimentType", text) + return "" + + +def metric_matches(metric_text: str, metric_tokens: set[str], pattern: str) -> bool: + if len(pattern) <= SHORT_TOKEN_LENGTH: + return pattern in metric_tokens + return pattern in metric_text + + +def metric_has_any_pattern(metric_text: str, metric_tokens: set[str], patterns: set[str]) -> bool: + return any(metric_matches(metric_text, metric_tokens, pattern) for pattern in patterns) + + +def sortable_value(value: Any) -> tuple[int, int, str]: + text = str(value or "").strip() + try: + return (0, int(text), "") + except ValueError: + return (1, 0, text) + + +def to_finite_float(value: Any) -> float | None: + if isinstance(value, bool): + return None + if isinstance(value, int | float): + converted = float(value) + return converted if math.isfinite(converted) else None + if isinstance(value, str): + prepared = value.strip().replace(",", ".") + if not prepared: + return None + try: + converted = float(prepared) + except ValueError: + return None + return converted if math.isfinite(converted) else None + return None + + +def is_nonzero_number(value: float | None) -> bool: + return value is not None and abs(value) > NONZERO_EPSILON + + +def is_invalid_metric_value(value: float) -> bool: + return value in INVALID_NUMERIC_SENTINELS + + +def resolve_output_path(input_path: Path, output_path: Path | None, output_name: str) -> Path: + if output_path is None: + return input_path.with_name(f"{input_path.stem}_{output_name}{input_path.suffix}") + if output_path.suffix.lower() == ".xlsx": + return output_path + return output_path / f"{input_path.stem}_{output_name}.xlsx" + + +def write_matrix(ws: Worksheet, rows: list[list[Any]]) -> None: + for row in rows: + ws.append(list(row)) + + +def style_table(ws: Worksheet, row_count: int, col_count: int) -> None: + if row_count <= 0 or col_count <= 0: + return + header_fill = PatternFill("solid", fgColor="D9EAF7") + for col in range(1, col_count + 1): + cell = ws.cell(row=1, column=col) + cell.font = Font(bold=True) + cell.fill = header_fill + width = min(max(len(str(cell.value or "")) + 2, 12), 45) + ws.column_dimensions[cell.column_letter].width = width + for row in ws.iter_rows(min_row=2, max_row=min(row_count, 200), max_col=col_count): + for cell in row: + if isinstance(cell.value, float): + cell.number_format = "0.000" + + +def unique_sheet_name(name: str, used_names: set[str]) -> str: + cleaned = sanitize_sheet_name(name) + original = cleaned + suffix = 1 + while cleaned in used_names: + suffix_text = f"_{suffix}" + cleaned = f"{original[: MAX_SHEET_NAME_LENGTH - len(suffix_text)]}{suffix_text}" + suffix += 1 + used_names.add(cleaned) + return cleaned + + +def sanitize_sheet_name(name: str) -> str: + cleaned = "".join("_" if char in r":\/?*[]" else char for char in str(name)).strip("'") + return (cleaned or "Sheet")[:MAX_SHEET_NAME_LENGTH] diff --git a/src/imgtests/reporting/distro_comparison_export.py b/src/imgtests/reporting/distro_comparison_export.py index 9b4ca710..2c690e0d 100644 --- a/src/imgtests/reporting/distro_comparison_export.py +++ b/src/imgtests/reporting/distro_comparison_export.py @@ -3,12 +3,10 @@ import argparse import logging import math -import re import statistics -from collections import defaultdict from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Any, Final, Literal, cast, get_args +from typing import TYPE_CHECKING, Any, Final from openpyxl import Workbook, load_workbook from openpyxl.chart import AreaChart, BarChart, LineChart, Reference, StockChart @@ -16,176 +14,60 @@ from openpyxl.chart.updown_bars import UpDownBars from openpyxl.styles import Font, PatternFill -from imgtests.database.models.experiment import ExperimentType +from imgtests.reporting.distro_comparison_common import ( + CHART_SHEET_PREFIX, + COMPARISON_DATA_SHEET, + COMPARISON_INDEX_SHEET, + COMPARISON_SELECTION_SHEET, + COUNT_PATTERNS, + DISTROS, + IPERF_PATTERNS, + MAX_SHEET_NAME_LENGTH, + MEMORY_PATTERNS, + PERCENT_PATTERNS, + STRESS_NG_PATTERNS, + THROUGHPUT_PATTERNS, + TIME_PATTERNS, + ComparisonGroup, + MetricBucket, + MetricExtractionOptions, + SheetRows, + build_comparison_groups, + build_configuration_distro_map, + build_experiment_info_map, + find_metric_columns, + is_comparison_sheet, + is_nonzero_number, + metric_has_any_pattern, + metric_matches, + normalize_header, + normalize_metric_text, + row_has_data, + sanitize_sheet_name, + sheet_has_data_rows, + style_table, + unique_sheet_name, + write_matrix, +) +from imgtests.reporting.distro_comparison_common import ( + extract_metric_buckets as extract_common_metric_buckets, +) +from imgtests.reporting.distro_comparison_common import ( + resolve_output_path as resolve_common_output_path, +) if TYPE_CHECKING: - from collections.abc import Iterator, Sequence + from collections.abc import Sequence from openpyxl.worksheet.worksheet import Worksheet logger = logging.getLogger(__name__) -DISTROS: Final = ("poky", "suse") -CONFIGURATION_SHEET: Final = "configuration" -EXPERIMENT_SHEET: Final = "experiment" COMPARISON_OUTPUT_NAME: Final = "comparison" -COMPARISON_SELECTION_SHEET: Final = "comparison_selection" -COMPARISON_INDEX_SHEET: Final = "comparison_index" -COMPARISON_DATA_SHEET: Final = "comparison_data" -COMPARISON_CHARTS_PREFIX: Final = "comparison_charts" -CHART_SHEET_PREFIX: Final = "charts" - -NONZERO_EPSILON: Final = 1e-12 -INVALID_NUMERIC_SENTINELS: Final = {-1.0} DEFAULT_MAX_CHARTS: Final = 0 DEFAULT_CHARTS_PER_SHEET: Final = 30 -SHORT_TOKEN_LENGTH: Final = 3 -EXPLICIT_PAIR_EXPERIMENT_COUNT: Final = 2 MIN_TREND_POINTS: Final = 3 -MAX_SHEET_NAME_LENGTH: Final = 31 -ID_COLUMN_SUFFIX: Final = "_id" -EXPERIMENT_TYPE_VALUES: Final = frozenset(get_args(ExperimentType)) - -SKIP_SOURCE_SHEETS: Final = { - COMPARISON_SELECTION_SHEET, - COMPARISON_INDEX_SHEET, - COMPARISON_DATA_SHEET, - COMPARISON_CHARTS_PREFIX, - "comparison_metrics", - "comparison_summary", -} - -LABEL_COLUMNS: Final = { - "configuration_id", - "distribution_description", - "experiment_id", - "run_result_id", - "id", - "test_name", - "tool_name", - "util_type", - "command", - "description", - "started_at", - "ended_at", - "type", - "config_id", - "tests_total", - "tests_passed", - "tests_failed", - "tests_broken", - "tests_skipped", -} - -TECHNICAL_TOKENS: Final = { - "id", - "pid", - "uid", - "gid", - "fd", - "rc", - "code", - "exit", - "status", - "returncode", - "return", - "retval", - "ret", - "socket", - "port", - "host", - "hostname", - "ip", - "addr", - "address", - "local", - "remote", - "timestamp", - "epoch", - "unix", - "version", - "command", - "cmd", - "cookie", - "mss", - "tos", -} - -TIME_PATTERNS: Final = { - "systemd", - "systemd_analyze", - "duration", - "elapsed", - "runtime", - "seconds", - "secs", - "time", - "latency", - "lat", - "clat", - "slat", - "rtt", - "wait", -} - -THROUGHPUT_PATTERNS: Final = { - "bps", - "bits_per_second", - "bitrate", - "bandwidth", - "bw", - "iops", - "ops_per_sec", - "ops_s", - "bogo_ops_s", - "pps", - "requests_per_second", -} - -PERCENT_PATTERNS: Final = { - "percent", - "percentage", - "pct", - "ratio", - "rate", - "util", - "usage", - "cpu_used", -} - -COUNT_PATTERNS: Final = { - "count", - "total", - "num", - "number", - "items", - "requests", - "retransmit", - "failed", - "errors", - "oom", - "broken", - "skipped", -} - -MEMORY_PATTERNS: Final = { - "rss", - "memory", - "mem", - "kb", - "mb", - "bytes", -} - -IPERF_PATTERNS: Final = { - "iperf", - "network_loopback", -} - -STRESS_NG_PATTERNS: Final = { - "stress_ng", -} CHART_SCORE_RULES: tuple[tuple[str, int], ...] = ( ("iperf", 140), @@ -215,32 +97,6 @@ ) -@dataclass(frozen=True) -class SheetRows: - name: str - headers: list[str] - rows: list[list[Any]] - - -@dataclass(frozen=True) -class ExperimentInfo: - experiment_id: int - distro: str - configuration_id: int - description: str - experiment_type: ExperimentType | Literal[""] - started_at: str - - -@dataclass(frozen=True) -class ComparisonGroup: - order: int - group_id: str - label: str - short_label: str - experiments: dict[str, ExperimentInfo] - - @dataclass(frozen=True) class DistributionStats: mean: float @@ -272,34 +128,10 @@ class ChartSpec: points: tuple[ChartPoint, ...] -@dataclass -class MetricBucket: - logical_test: str - source_sheet: str - metric_name: str - values: dict[str, dict[str, list[float]]] - - -@dataclass(frozen=True) -class MetricWorksheetContext: - sheet_name: str - source_sheet: str - rows: Iterator[tuple[Any, ...]] - headers: list[str] - config_index: int - experiment_index: int - metric_indexes: list[int] - test_name_index: int | None - tool_name_index: int | None - util_type_index: int | None - distribution_index: int | None - - @dataclass(frozen=True) class DistroComparisonExportOptions: output_path: Path | None = None experiment_ids: Sequence[str] | None = None - latest_pair_only: bool = False max_charts: int = DEFAULT_MAX_CHARTS charts_per_sheet: int = DEFAULT_CHARTS_PER_SHEET copy_source_sheets: bool = False @@ -396,7 +228,6 @@ def build_report( experiment_ids=( list(options.experiment_ids) if options.experiment_ids is not None else None ), - latest_pair_only=options.latest_pair_only, ) logger.info("Comparable Poky/SUSE groups: %s", len(groups)) if groups: @@ -432,7 +263,11 @@ def build_report( ws = result_wb.create_sheet("no_data") ws["A1"] = "No data found." - result_path = resolve_output_path(input_path, options.output_path) + result_path = resolve_common_output_path( + input_path, + options.output_path, + COMPARISON_OUTPUT_NAME, + ) result_path.parent.mkdir(parents=True, exist_ok=True) result_wb.save(result_path) return result_path @@ -468,367 +303,19 @@ def read_source_sheets(workbook: Workbook) -> list[SheetRows]: return sheets -def is_comparison_sheet(sheet_name: str) -> bool: - return sheet_name in SKIP_SOURCE_SHEETS or sheet_name.startswith( - (f"{COMPARISON_CHARTS_PREFIX}_", f"{CHART_SHEET_PREFIX}_"), - ) - - -def build_configuration_distro_map(workbook: Workbook) -> dict[str, str]: - if CONFIGURATION_SHEET not in workbook.sheetnames: - return {} - ws = workbook[CONFIGURATION_SHEET] - rows = ws.iter_rows(values_only=True) - try: - headers = [normalize_header(value) for value in next(rows)] - except StopIteration: - return {} - - config_index = column_index(headers, "configuration_id") - distro_index = column_index(headers, "distribution_description") - if config_index is None or distro_index is None: - return {} - - result: dict[str, str] = {} - for row in rows: - key = id_key(cell_value(row, config_index)) - distro = normalize_distro(cell_value(row, distro_index)) - if key and distro: - result[key] = distro - return result - - -def build_experiment_info_map( - workbook: Workbook, - config_distro: dict[str, str], -) -> dict[int, ExperimentInfo]: - if EXPERIMENT_SHEET not in workbook.sheetnames: - return {} - ws = workbook[EXPERIMENT_SHEET] - rows = ws.iter_rows(values_only=True) - try: - headers = [normalize_header(value) for value in next(rows)] - except StopIteration: - return {} - - config_index = column_index(headers, "configuration_id") - experiment_index = column_index(headers, "experiment_id") - description_index = column_index(headers, "description") - type_index = column_index(headers, "type") - started_at_index = column_index(headers, "started_at") - if config_index is None or experiment_index is None: - return {} - - result: dict[int, ExperimentInfo] = {} - for row in rows: - experiment_id = int_id_or_none(cell_value(row, experiment_index)) - configuration_id = int_id_or_none(cell_value(row, config_index)) - if experiment_id is None or configuration_id is None: - continue - result[experiment_id] = ExperimentInfo( - experiment_id=experiment_id, - distro=config_distro.get(id_key(configuration_id), ""), - configuration_id=configuration_id, - description=str(cell_value(row, description_index) or "").strip(), - experiment_type=normalize_experiment_type(cell_value(row, type_index)), - started_at=str(cell_value(row, started_at_index) or "").strip(), - ) - return result - - -def build_comparison_groups( - experiment_info: dict[int, ExperimentInfo], - *, - experiment_ids: list[str] | None, - latest_pair_only: bool, -) -> list[ComparisonGroup]: - selected_ids = normalize_selected_ids(experiment_ids) - if experiment_ids and not selected_ids: - logger.warning("No valid integer experiment IDs were provided.") - return [] - - if selected_ids and len(selected_ids) == EXPLICIT_PAIR_EXPERIMENT_COUNT: - ordered_ids = sorted(selected_ids) - infos = [experiment_info.get(experiment_id) for experiment_id in ordered_ids] - if not all(info is not None for info in infos): - missing = [ - str(experiment_id) - for experiment_id, info in zip(ordered_ids, infos, strict=True) - if info is None - ] - logger.warning("Experiment IDs not found in experiment sheet: %s", ", ".join(missing)) - return [] - by_distro = { - info.distro: info for info in infos if info is not None and info.distro in DISTROS - } - if not all(distro in by_distro for distro in DISTROS): - logger.warning( - "Two IDs were provided, but they are not one Poky and one SUSE experiment.", - ) - return [] - return [make_group(by_distro["poky"], by_distro["suse"], order=1)] - - filtered_infos = [ - info - for info in experiment_info.values() - if info.distro in DISTROS and (not selected_ids or info.experiment_id in selected_ids) - ] - grouped: dict[tuple[str, str], dict[str, list[ExperimentInfo]]] = defaultdict( - lambda: defaultdict(list), - ) - for info in filtered_infos: - grouped[(info.description, info.experiment_type)][info.distro].append(info) - - groups: list[ComparisonGroup] = [] - order = 1 - for key in sorted(grouped, key=lambda item: (item[0], item[1])): - by_distro = grouped[key] - if not all(distro in by_distro for distro in DISTROS): - continue - poky_items = sorted(by_distro["poky"], key=experiment_sort_key) - suse_items = sorted(by_distro["suse"], key=experiment_sort_key) - pair_count = min(len(poky_items), len(suse_items)) - if len(poky_items) != len(suse_items): - logger.warning( - "Experiment group '%s/%s' has %s Poky and %s SUSE runs; " - "only %s comparable pairs will be used.", - key[0], - key[1], - len(poky_items), - len(suse_items), - pair_count, - ) - for poky_info, suse_info in zip( - poky_items[:pair_count], - suse_items[:pair_count], - strict=True, - ): - groups.append(make_group(poky_info, suse_info, order=order)) - order += 1 - - if latest_pair_only and groups: - return [ - max( - groups, - key=lambda group: max( - experiment_sort_key(info) for info in group.experiments.values() - ), - ), - ] - return groups - - -def make_group( - poky_info: ExperimentInfo, - suse_info: ExperimentInfo, - *, - order: int, -) -> ComparisonGroup: - group_id = f"exp_{poky_info.experiment_id}_{suse_info.experiment_id}" - label_parts = [part for part in (poky_info.description, poky_info.experiment_type) if part] - base_label = " | ".join(label_parts) or "experiment" - short_label = f"{order}: p{poky_info.experiment_id}/s{suse_info.experiment_id}" - label = ( - f"{base_label} | exp {poky_info.experiment_id} / poky vs " - f"exp {suse_info.experiment_id} / suse" - ) - return ComparisonGroup( - order=order, - group_id=group_id, - label=label, - short_label=short_label, - experiments={"poky": poky_info, "suse": suse_info}, - ) - - -def normalize_selected_ids(experiment_ids: list[str] | None) -> set[int]: - if not experiment_ids: - return set() - selected_ids: set[int] = set() - invalid_ids: list[str] = [] - for experiment_id in experiment_ids: - normalized_id = int_id_or_none(experiment_id) - if normalized_id is None: - invalid_ids.append(str(experiment_id)) - continue - selected_ids.add(normalized_id) - - if invalid_ids: - logger.warning("Experiment IDs must be integers and were ignored: %s", invalid_ids) - return selected_ids - - -def experiment_sort_key(info: ExperimentInfo) -> tuple[str, int]: - return info.started_at, info.experiment_id - - def extract_metric_buckets( - workbook: Workbook, + workbook: Any, config_distro: dict[str, str], groups: list[ComparisonGroup], ) -> list[MetricBucket]: - group_by_experiment = build_group_by_experiment(groups) - buckets: dict[tuple[str, str, str], MetricBucket] = {} - - for worksheet in workbook.worksheets: - context = metric_worksheet_context(worksheet) - if context is None: - continue - for row in context.rows: - collect_metric_row(row, context, group_by_experiment, config_distro, buckets) - - return list(buckets.values()) - - -def build_group_by_experiment(groups: list[ComparisonGroup]) -> dict[str, ComparisonGroup]: - group_by_experiment: dict[str, ComparisonGroup] = {} - for group in groups: - for info in group.experiments.values(): - group_by_experiment[id_key(info.experiment_id)] = group - return group_by_experiment - - -def metric_worksheet_context(worksheet: Worksheet) -> MetricWorksheetContext | None: - sheet_name = worksheet.title - if is_comparison_sheet(sheet_name): - return None - - rows_iter = worksheet.iter_rows(values_only=True) - try: - headers = [normalize_header(value) for value in next(rows_iter)] - except StopIteration: - return None - - config_index = column_index(headers, "configuration_id") - experiment_index = column_index(headers, "experiment_id") - if config_index is None or experiment_index is None: - return None - - metric_indexes = find_metric_columns(headers) - if not metric_indexes: - return None - - return MetricWorksheetContext( - sheet_name=sheet_name, - source_sheet=strip_excel_suffix(sheet_name), - rows=rows_iter, - headers=headers, - config_index=config_index, - experiment_index=experiment_index, - metric_indexes=metric_indexes, - test_name_index=column_index(headers, "test_name"), - tool_name_index=column_index(headers, "tool_name"), - util_type_index=column_index(headers, "util_type"), - distribution_index=column_index(headers, "distribution_description"), + return extract_common_metric_buckets( + workbook, + config_distro, + groups, + MetricExtractionOptions(column_selector=find_metric_columns), ) -def collect_metric_row( - row: tuple[Any, ...], - context: MetricWorksheetContext, - group_by_experiment: dict[str, ComparisonGroup], - config_distro: dict[str, str], - buckets: dict[tuple[str, str, str], MetricBucket], -) -> None: - experiment_id = id_key(cell_value(row, context.experiment_index)) - group = group_by_experiment.get(experiment_id) - if group is None: - return - - distro = row_distro(row, context.distribution_index, context.config_index, config_distro) - if distro not in DISTROS: - return - - logical_test = logical_test_name( - context.sheet_name, - cell_value(row, context.test_name_index), - cell_value(row, context.tool_name_index), - cell_value(row, context.util_type_index), - ) - - for metric_index in context.metric_indexes: - metric_name = context.headers[metric_index] - value = to_finite_float(cell_value(row, metric_index)) - if value is None or is_invalid_metric_value(value) or not is_nonzero_number(value): - continue - bucket = get_or_create_metric_bucket( - buckets, - logical_test, - context.source_sheet, - metric_name, - ) - bucket.values[group.group_id][distro].append(value) - - -def get_or_create_metric_bucket( - buckets: dict[tuple[str, str, str], MetricBucket], - logical_test: str, - source_sheet: str, - metric_name: str, -) -> MetricBucket: - bucket_key = (logical_test, source_sheet, metric_name) - bucket = buckets.get(bucket_key) - if bucket is None: - bucket = MetricBucket( - logical_test=logical_test, - source_sheet=source_sheet, - metric_name=metric_name, - values=defaultdict(lambda: defaultdict(list)), - ) - buckets[bucket_key] = bucket - return bucket - - -def find_metric_columns(headers: list[str]) -> list[int]: - metric_indexes: list[int] = [] - for index, header in enumerate(headers): - if not header or header in LABEL_COLUMNS or header.endswith(ID_COLUMN_SUFFIX): - continue - normalized_header = normalize_metric_text(header) - if not normalized_header: - continue - tokens = set(normalized_header.split("_")) - if tokens & TECHNICAL_TOKENS: - continue - metric_indexes.append(index) - return metric_indexes - - -def row_distro( - row: tuple[Any, ...], - distribution_index: int | None, - config_index: int, - config_distro: dict[str, str], -) -> str: - if distribution_index is not None: - distro = normalize_distro(cell_value(row, distribution_index)) - if distro: - return distro - return config_distro.get(id_key(cell_value(row, config_index)), "") - - -def logical_test_name(sheet_name: str, test_name: Any, tool_name: Any, util_type: Any) -> str: - parts: list[str] = [] - prepared_test_name = str(test_name or "").strip() - prepared_tool_name = str(tool_name or "").strip() - prepared_util_type = str(util_type or "").strip() - - if prepared_test_name: - parts.append(prepared_test_name) - else: - parts.append(strip_excel_suffix(sheet_name)) - - if prepared_tool_name and prepared_tool_name.lower() not in prepared_test_name.lower(): - parts.append(prepared_tool_name) - if prepared_util_type: - parts.append(prepared_util_type) - return " / ".join(parts) - - -def strip_excel_suffix(sheet_name: str) -> str: - return re.sub(r"_\d+$", "", str(sheet_name).strip()) - - def build_chart_specs( buckets: list[MetricBucket], groups: list[ComparisonGroup], @@ -1323,159 +810,6 @@ def write_styled_table(ws: Worksheet, rows: list[list[Any]]) -> None: ws.auto_filter.ref = ws.dimensions -def write_matrix(ws: Worksheet, rows: list[list[Any]]) -> None: - for row in rows: - ws.append(list(row)) - - -def style_table(ws: Worksheet, row_count: int, col_count: int) -> None: - if row_count <= 0 or col_count <= 0: - return - header_fill = PatternFill("solid", fgColor="D9EAF7") - for col in range(1, col_count + 1): - cell = ws.cell(row=1, column=col) - cell.font = Font(bold=True) - cell.fill = header_fill - width = min(max(len(str(cell.value or "")) + 2, 12), 45) - ws.column_dimensions[cell.column_letter].width = width - for row in ws.iter_rows(min_row=2, max_row=min(row_count, 200), max_col=col_count): - for cell in row: - if isinstance(cell.value, float): - cell.number_format = "0.000" - - -def resolve_output_path(input_path: Path, output_path: Path | None) -> Path: - if output_path is None: - return input_path.with_name( - f"{input_path.stem}_{COMPARISON_OUTPUT_NAME}{input_path.suffix}", - ) - if output_path.suffix.lower() == ".xlsx": - return output_path - return output_path / f"{input_path.stem}_{COMPARISON_OUTPUT_NAME}.xlsx" - - -def normalize_header(value: Any) -> str: - return "" if value is None else str(value).strip() - - -def column_index(headers: list[str], name: str) -> int | None: - for index, header in enumerate(headers): - if header == name: - return index - return None - - -def cell_value(row: tuple[Any, ...] | list[Any], index: int | None) -> Any: - if index is None or index >= len(row): - return None - return row[index] - - -def row_has_data(row: tuple[Any, ...] | list[Any]) -> bool: - return any(not is_empty_value(value) for value in row) - - -def sheet_has_data_rows(sheet: SheetRows) -> bool: - return any(row_has_data(row) for row in sheet.rows) - - -def is_empty_value(value: Any) -> bool: - return value is None or value == "" - - -def id_key(value: Any) -> str: - if isinstance(value, bool) or value is None: - return "" - if isinstance(value, int): - return str(value) - if isinstance(value, float) and value.is_integer(): - return str(int(value)) - return str(value).strip() - - -def int_id_or_none(value: Any) -> int | None: - if isinstance(value, bool) or value is None: - return None - if isinstance(value, int): - return value - if isinstance(value, float): - return int(value) if value.is_integer() else None - - text = str(value).strip() - if not text: - return None - try: - return int(text) - except ValueError: - return None - - -def normalize_distro(value: Any) -> str: - text = str(value or "").strip().lower() - if "suse" in text: - return "suse" - if "yocto" in text or "poky" in text: - return "poky" - return text - - -def normalize_metric_text(value: str) -> str: - normalized = re.sub(r"[^a-zA-Z0-9]+", "_", str(value).lower()) - normalized = re.sub(r"_+", "_", normalized) - return normalized.strip("_") - - -def normalize_experiment_type(value: Any) -> ExperimentType | Literal[""]: - text = str(value or "").strip() - if text in EXPERIMENT_TYPE_VALUES: - return cast("ExperimentType", text) - return "" - - -def metric_matches(metric_text: str, metric_tokens: set[str], pattern: str) -> bool: - if len(pattern) <= SHORT_TOKEN_LENGTH: - return pattern in metric_tokens - return pattern in metric_text - - -def metric_has_any_pattern(metric_text: str, metric_tokens: set[str], patterns: set[str]) -> bool: - return any(metric_matches(metric_text, metric_tokens, pattern) for pattern in patterns) - - -def sortable_value(value: Any) -> tuple[int, int, str]: - text = str(value or "").strip() - try: - return (0, int(text), "") - except ValueError: - return (1, 0, text) - - -def to_finite_float(value: Any) -> float | None: - if isinstance(value, bool): - return None - if isinstance(value, int | float): - converted = float(value) - return converted if math.isfinite(converted) else None - if isinstance(value, str): - prepared = value.strip().replace(",", ".") - if not prepared: - return None - try: - converted = float(prepared) - except ValueError: - return None - return converted if math.isfinite(converted) else None - return None - - -def is_nonzero_number(value: float | None) -> bool: - return value is not None and abs(value) > NONZERO_EPSILON - - -def is_invalid_metric_value(value: float) -> bool: - return value in INVALID_NUMERIC_SENTINELS - - def distribution_stats_or_none(values: list[float]) -> DistributionStats | None: prepared = sorted(value for value in values if is_nonzero_number(value)) if not prepared: @@ -1509,23 +843,6 @@ def percentile(sorted_values: list[float], fraction: float) -> float: return lower_value + (upper_value - lower_value) * weight -def unique_sheet_name(name: str, used_names: set[str]) -> str: - cleaned = sanitize_sheet_name(name) - original = cleaned - suffix = 1 - while cleaned in used_names: - suffix_text = f"_{suffix}" - cleaned = f"{original[: MAX_SHEET_NAME_LENGTH - len(suffix_text)]}{suffix_text}" - suffix += 1 - used_names.add(cleaned) - return cleaned - - -def sanitize_sheet_name(name: str) -> str: - cleaned = "".join("_" if char in r":\/?*[]" else char for char in str(name)).strip("'") - return (cleaned or "Sheet")[:MAX_SHEET_NAME_LENGTH] - - def main() -> None: logging.basicConfig(level=logging.INFO, format="%(message)s") args = parse_args() @@ -1534,7 +851,6 @@ def main() -> None: options=DistroComparisonExportOptions( output_path=args.output, experiment_ids=args.experiment_ids, - latest_pair_only=args.latest_pair_only, max_charts=args.max_charts, charts_per_sheet=args.charts_per_sheet, copy_source_sheets=args.copy_source_sheets, diff --git a/src/imgtests/reporting/distro_comparison_status_export.py b/src/imgtests/reporting/distro_comparison_status_export.py new file mode 100644 index 00000000..0b301b9d --- /dev/null +++ b/src/imgtests/reporting/distro_comparison_status_export.py @@ -0,0 +1,1271 @@ +from __future__ import annotations + +import logging +import re +import statistics +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Final + +from openpyxl import Workbook, load_workbook +from openpyxl.styles import PatternFill + +from imgtests.reporting.distro_comparison_common import ( + COUNT_PATTERNS, + IPERF_PATTERNS, + LABEL_COLUMNS, + MEMORY_PATTERNS, + NONZERO_EPSILON, + PERCENT_PATTERNS, + STRESS_NG_PATTERNS, + THROUGHPUT_PATTERNS, + TIME_PATTERNS, + ComparisonGroup, + MetricBucket, + MetricExtractionOptions, + build_comparison_groups, + build_configuration_distro_map, + build_experiment_info_map, + metric_has_any_pattern, + normalize_metric_text, + style_table, + unique_sheet_name, + write_matrix, +) +from imgtests.reporting.distro_comparison_common import ( + extract_metric_buckets as extract_common_metric_buckets, +) +from imgtests.reporting.distro_comparison_common import ( + resolve_output_path as resolve_common_output_path, +) + +if TYPE_CHECKING: + from collections.abc import Sequence + from pathlib import Path + + from openpyxl.worksheet.worksheet import Worksheet + +logger = logging.getLogger(__name__) + +BASELINE_DISTRO: Final = "suse" +COMPARED_DISTRO: Final = "poky" +OUTPUT_NAME: Final = "comparison_status" +SUMMARY_SHEET: Final = "summary" +TEST_STATUS_SHEET: Final = "test_status" +METRIC_STATUS_SHEET: Final = "metric_status" + +DEFAULT_EPSILON_PERCENT: Final = 1.0 + +HIGHER_IS_BETTER: Final = "higher" +LOWER_IS_BETTER: Final = "lower" +NOT_COMPARABLE_DIRECTION: Final = "not_comparable" + +STATUS_OK: Final = "PASS" +STATUS_FAIL: Final = "FAIL" +STATUS_NOT_COMPARABLE: Final = "NOT_COMPARABLE" + +CHANGE_IMPROVED: Final = "improved" +CHANGE_DEGRADED: Final = "degraded" +CHANGE_UNCHANGED: Final = "unchanged" +CHANGE_NOT_COMPARABLE: Final = "not_comparable" + +RAW_INCREASED: Final = "increased" +RAW_DECREASED: Final = "decreased" +RAW_UNCHANGED: Final = "unchanged" +RAW_UNAVAILABLE: Final = "unavailable" + +FLATTENED_DUPLICATE_SUFFIX_RE: Final = re.compile(r"_\d+$") +PTS_TEST_RUN_TIMES_METRIC: Final = "results_test_run_times" +PTS_TEST_RUN_TIMES_RE: Final = re.compile(r"^results_test_run_times(?:_\d+)?$") + +IPERF_INTERVAL_ANY_RE: Final = re.compile(r"(^|_)intervals(_|$)") + +IPERF_PREFIX_RE: Final = re.compile(r"^iperf3_") +IPERF_TARGET_METRIC_RE: Final = re.compile( + r"^(" + r"(client|server)_(sum|sent|received|streams_sender|streams_receiver|streams_udp)_" + r"(bps|bytes|packets|retransmits|rtt|mean_rtt|min_rtt|max_rtt|jitter_ms|" + r"lost_packets|lost_percent|out_of_order)" + r")$", +) +IPERF_LOWER_IS_BETTER_RE: Final = re.compile( + r"(^|_)(jitter_ms|lost_packets|lost_percent|out_of_order|" + r"retransmits|rtt|mean_rtt|min_rtt|max_rtt)$", +) +IPERF_CPU_HOST_METRIC_RE: Final = re.compile( + r"^(iperf3_)?(client|server)_cpu_host_(total|user|system)$", +) +IPERF_TECHNICAL_METRIC_PARTS: Final = { + "pmtu", + "rcvbuf_actual", + "seconds", + "snd_cwnd", + "snd_wnd", + "sndbuf_actual", + "sock_bufsize", + "target_bitrate", + "test_bidir", + "test_block_size", + "test_blocks", + "test_bytes", + "test_duration_sec", + "test_interval", + "test_omit", + "test_reverse", + "test_streams", + "test_target_bitrate", +} + +NON_PERFORMANCE_SOURCE_SHEETS: Final = { + "planned_stage", + "task_result_for_stage_warmup", + "unnamed_util_run_result", +} + +STRESS_NG_TARGET_METRICS: Final = { + "bogo_ops", + "bogo_ops_s_real_time", + "bogo_ops_s_usr_sys_time", + "stress_ng_metrics_bogo_ops", + "stress_ng_metrics_bogo_ops_s_real_time", + "stress_ng_metrics_bogo_ops_s_usr_sys_time", +} + +PERF_TARGET_METRICS: Final = { + "gb_per_sec_default", + "gb_per_sec_movsq_based", + "gb_per_sec_unrolled", + "ops_per_sec", + "usecs_per_op", + "perf_metrics_gb_per_sec_default", + "perf_metrics_gb_per_sec_movsq_based", + "perf_metrics_gb_per_sec_unrolled", + "perf_metrics_ops_per_sec", + "perf_metrics_usecs_per_op", +} + +SYSTEMD_TARGET_METRICS: Final = { + "firmware_time", + "loader_time", + "initrd_time", + "kernel_time", + "userspace_time", + "total_time", +} + +FIO_TARGET_METRIC_RE: Final = re.compile( + r"^(read|write|trim)_(" + r"bw|bw_bytes|iops|io_bytes|io_kbytes|total_ios|" + r"(slat|clat|lat)_(ns|us)_(min|max|mean|stddev)" + r")$", +) +FIO_LOWER_IS_BETTER_RE: Final = re.compile( + r"^(read|write|trim)_(slat|clat|lat)_(ns|us)_(min|max|mean|stddev)$", +) + +LOWER_IS_BETTER_TARGETS: Final = { + "firmware_time", + "loader_time", + "initrd_time", + "kernel_time", + "userspace_time", + "total_time", + "usecs_per_op", + "perf_metrics_usecs_per_op", +} + +LOWER_IS_BETTER_PATTERNS: Final = { + "broken", + "clat", + "drop", + "dropped", + "elapsed", + "error", + "errors", + "fail", + "failed", + "failure", + "failures", + "fault", + "faults", + "jitter", + "jitter_ms", + "latency", + "lost", + "lost_packets", + "lost_percent", + "oom", + "out_of_order", + "context_switches", + "retransmit", + "retransmits", + "retry", + "retries", + "rtt", + "skip", + "skipped", + "slat", + "timeout", + "timeouts", + "untrustworthy", +} + +HIGHER_IS_BETTER_PATTERNS: Final = { + "bandwidth", + "bitrate", + "bits_per_second", + "bogo_ops", + "bogo_ops_s", + "bps", + "gb_per_sec", + "iops", + "io_bytes", + "io_kbytes", + "ops_per_sec", + "ops_s", + "packets", + "passed", + "passrate", + "pps", + "requests", + "requests_per_second", + "success", + "successes", + "total_ios", +} + +RESOURCE_USAGE_PATTERNS: Final = { + "cpu", + "cpu_used", + "host_system", + "host_total", + "host_user", + "remote_system", + "remote_total", + "remote_user", + "rss", + "usage", + "util", +} + +TECHNICAL_OR_ID_PATTERNS: Final = { + "addr", + "address", + "code", + "cookie", + "fd", + "gid", + "host", + "hostname", + "id", + "ip", + "local", + "mss", + "pid", + "port", + "rc", + "remote", + "return", + "returncode", + "retval", + "socket", + "status", + "timestamp", + "tos", + "uid", + "unix", + "version", +} + + +@dataclass(frozen=True) +class DistroComparisonStatusOptions: + output_path: Path | None = None + experiment_ids: Sequence[str] | None = None + epsilon_percent: float = DEFAULT_EPSILON_PERCENT + + +@dataclass(frozen=True) +class MetricDirection: + better_direction: str + interpretation: str + reason: str + comparable: bool + + +@dataclass(frozen=True) +class MeanStats: + mean: float + sample_count: int + + +@dataclass(frozen=True) +class MetricStatusRow: + status: str + performance_change: str + raw_value_change: str + group_order: int + group_label: str + source_sheet: str + test_name: str + metric_name: str + baseline_distro: str + baseline_mean: float + baseline_sample_count: int + compared_distro: str + compared_mean: float + compared_sample_count: int + better_direction: str + epsilon_percent: float + value_delta_percent: float | None + performance_delta_percent: float | None + + +@dataclass(frozen=True) +class TestStatusRow: + status: str + group_order: int + group_label: str + source_sheet: str + test_name: str + baseline_distro: str + compared_distro: str + epsilon_percent: float + metrics_total: int + passed_metrics: int + failed_metrics: int + mean_value_delta_percent: float | None + mean_performance_delta_percent: float | None + failed_metric_names: str + + +def export_distro_comparison_status_to_excel( + input_path: Path, + options: DistroComparisonStatusOptions | None = None, +) -> Path: + options = options or DistroComparisonStatusOptions() + if not options.experiment_ids: + msg = "Experiment IDs are required for comparison status export." + raise ValueError(msg) + + source_wb = load_workbook(input_path, read_only=True, data_only=True) + config_distro = build_configuration_distro_map(source_wb) + experiment_info = build_experiment_info_map(source_wb, config_distro) + groups = build_comparison_groups( + experiment_info, + experiment_ids=list(options.experiment_ids), + ) + logger.info("Comparable Poky/SUSE groups: %s", len(groups)) + + rows: list[MetricStatusRow] = [] + if groups: + buckets = extract_status_metric_buckets(source_wb, config_distro, groups) + logger.info("Metric buckets with data: %s", len(buckets)) + rows = build_metric_status_rows( + buckets, + groups, + epsilon_percent=options.epsilon_percent, + ) + logger.info("Metric status rows: %s", len(rows)) + + result_path = resolve_common_output_path(input_path, options.output_path, OUTPUT_NAME) + write_status_workbook( + result_path, + rows, + groups, + epsilon_percent=max(0.0, options.epsilon_percent), + ) + return result_path + + +def extract_status_metric_buckets( + workbook: Any, + config_distro: dict[str, str], + groups: list[ComparisonGroup], +) -> list[MetricBucket]: + return extract_common_metric_buckets( + workbook, + config_distro, + groups, + MetricExtractionOptions( + column_selector=status_metric_columns, + name_mapper=status_metric_bucket_name, + predicate=should_collect_metric_status, + require_nonzero=False, + ), + ) + + +def status_metric_columns(headers: list[str]) -> list[int]: + metric_indexes: list[int] = [] + for index, header in enumerate(headers): + if not header or header in LABEL_COLUMNS: + continue + if normalize_metric_text(header): + metric_indexes.append(index) + return metric_indexes + + +def should_collect_metric_status( + source_sheet: str, + logical_test: str, + metric_name: str, +) -> bool: + return target_metric_kind_for(source_sheet, logical_test, metric_name) is not None + + +def build_metric_status_rows( + buckets: list[MetricBucket], + groups: list[ComparisonGroup], + *, + epsilon_percent: float, +) -> list[MetricStatusRow]: + normalized_epsilon = max(0.0, epsilon_percent) + rows: list[MetricStatusRow] = [] + sorted_buckets = sorted( + buckets, + key=lambda bucket: (bucket.source_sheet, bucket.logical_test, bucket.metric_name), + ) + + for bucket in sorted_buckets: + if not should_write_metric_status(bucket): + continue + direction = metric_direction(bucket) + if not direction.comparable: + continue + for group in groups: + values_by_distro = bucket.values.get(group.group_id, {}) + baseline_stats = mean_stats_or_none(values_by_distro.get(BASELINE_DISTRO, [])) + compared_stats = mean_stats_or_none(values_by_distro.get(COMPARED_DISTRO, [])) + if baseline_stats is None or compared_stats is None: + continue + + value_delta = percent_delta(compared_stats.mean, baseline_stats.mean) + performance_delta = directional_percent_delta( + value_delta, + direction.better_direction, + ) + performance_change = performance_change_for_values( + baseline_stats.mean, + compared_stats.mean, + direction, + normalized_epsilon, + ) + rows.append( + MetricStatusRow( + status=status_for(performance_change), + performance_change=performance_change, + raw_value_change=raw_value_change_for( + value_delta, + normalized_epsilon, + baseline=baseline_stats.mean, + compared=compared_stats.mean, + ), + group_order=group.order, + group_label=group.short_label, + source_sheet=bucket.source_sheet, + test_name=bucket.logical_test, + metric_name=bucket.metric_name, + baseline_distro=BASELINE_DISTRO, + baseline_mean=baseline_stats.mean, + baseline_sample_count=baseline_stats.sample_count, + compared_distro=COMPARED_DISTRO, + compared_mean=compared_stats.mean, + compared_sample_count=compared_stats.sample_count, + better_direction=direction.better_direction, + epsilon_percent=normalized_epsilon, + value_delta_percent=value_delta, + performance_delta_percent=performance_delta, + ), + ) + + return rows + + +def should_write_metric_status(bucket: MetricBucket) -> bool: + return target_metric_kind(bucket) is not None + + +def target_metric_kind(bucket: MetricBucket) -> str | None: + return target_metric_kind_for(bucket.source_sheet, bucket.logical_test, bucket.metric_name) + + +def target_metric_kind_for(source_sheet: str, logical_test: str, metric_name: str) -> str | None: + metric_text = normalize_metric_text(metric_name) + source_text = normalize_metric_text(source_sheet) + test_text = normalize_metric_text(logical_test) + + if is_garbage_metric(source_text, test_text, metric_text): + return None + + direction = inferred_metric_direction(source_text, test_text, metric_text) + if not direction.comparable: + return None + + return metric_kind_for(source_text, test_text, metric_text) + + +def is_garbage_metric(source_text: str, _test_text: str, metric_text: str) -> bool: + if source_text in NON_PERFORMANCE_SOURCE_SHEETS: + return True + if is_flattened_duplicate_metric(metric_text) and not is_pts_test_run_times_metric( + source_text, + metric_text, + ): + return True + if IPERF_INTERVAL_ANY_RE.search(metric_text): + return True + if has_technical_iperf_metric_part(normalized_iperf_metric_name(metric_text)): + return True + if is_iperf_cpu_host_metric(metric_text): + return False + return bool(technical_metric_direction(set(metric_text.split("_")))) + + +def metric_kind_for(source_text: str, test_text: str, metric_text: str) -> str: + if is_iperf_metric_context_text(source_text, test_text, metric_text): + return "iperf3" + if is_perf_metric_context_text(source_text, test_text, metric_text): + return "perf" + if is_stress_ng_metric_context_text(source_text, test_text, metric_text): + return "stress_ng" + if is_fio_metric_context_text(source_text, test_text): + return "fio" + if source_text: + return source_text + return "inferred" + + +def inferred_metric_direction( + source_text: str, + test_text: str, + metric_text: str, +) -> MetricDirection: + text = normalize_metric_text(f"{source_text} {test_text} {metric_text}") + tokens = set(text.split("_")) + + comparable = context_specific_metric_direction(source_text, test_text, metric_text) + if comparable is not None: + return comparable + + comparable = comparable_explicit_direction(text, tokens) + if comparable is not None: + return comparable + + comparable = generic_context_direction(text, tokens) + if comparable is not None: + return comparable + + non_comparable = non_comparable_explicit_direction(text, tokens) + if non_comparable is not None: + return non_comparable + + return MetricDirection( + better_direction=NOT_COMPARABLE_DIRECTION, + interpretation="unknown metric direction", + reason="the metric is numeric but higher/lower cannot be inferred safely", + comparable=False, + ) + + +def context_specific_metric_direction( # noqa: PLR0911 + source_text: str, + test_text: str, + metric_text: str, +) -> MetricDirection | None: + normalized_metric = normalized_target_metric_name(metric_text) + metric_tokens = set(normalized_metric.split("_")) + + if is_iperf_metric_context_text(source_text, test_text, metric_text): + if IPERF_LOWER_IS_BETTER_RE.search(normalized_metric): + return comparable_direction( + LOWER_IS_BETTER, + "iperf quality/error metric", + "lower values mean less loss, jitter, retransmits, RTT, or packet disorder", + ) + if is_iperf_cpu_host_metric(metric_text): + return comparable_direction( + LOWER_IS_BETTER, + "iperf host CPU usage metric", + "lower values mean the same network work used less host CPU", + ) + if metric_has_any_pattern(normalized_metric, metric_tokens, {"bps", "bytes", "packets"}): + return comparable_direction( + HIGHER_IS_BETTER, + "iperf throughput metric", + "higher values mean more network throughput or transferred data", + ) + + if is_fio_metric_context_text(source_text, test_text): + if FIO_LOWER_IS_BETTER_RE.fullmatch(normalized_metric): + return comparable_direction( + LOWER_IS_BETTER, + "fio latency metric", + "lower values mean less IO latency", + ) + if metric_has_any_pattern( + normalized_metric, + metric_tokens, + {"bw", "io_bytes", "io_kbytes", "iops", "total_ios"}, + ): + return comparable_direction( + HIGHER_IS_BETTER, + "fio throughput metric", + "higher values mean more IO completed", + ) + + if is_perf_metric_context_text(source_text, test_text, metric_text): + if normalized_metric in {"usecs_per_op", "perf_metrics_usecs_per_op"}: + return comparable_direction( + LOWER_IS_BETTER, + "perf latency metric", + "lower values mean each operation finished sooner", + ) + if normalized_metric in PERF_TARGET_METRICS: + return comparable_direction( + HIGHER_IS_BETTER, + "perf throughput metric", + "higher values mean more benchmark work completed", + ) + + return None + + +def is_iperf_metric_context_text(source_text: str, test_text: str, metric_text: str) -> bool: + return ( + "iperf3" in source_text + or "iperf" in source_text + or "iperf3" in test_text + or "iperf" in test_text + or metric_text.startswith("iperf3_") + ) + + +def is_stress_ng_metric_context_text(source_text: str, test_text: str, metric_text: str) -> bool: + return ( + source_text.startswith("stress_ng") + or "stress_ng" in test_text + or metric_text.startswith("stress_ng_metrics_") + ) + + +def is_perf_metric_context_text(source_text: str, test_text: str, metric_text: str) -> bool: + return source_text == "perf" or "perf" in test_text or metric_text.startswith("perf_metrics_") + + +def is_fio_metric_context_text(source_text: str, test_text: str) -> bool: + return source_text.startswith("fio") or "fio" in test_text + + +def status_metric_bucket_name(source_sheet: str, logical_test: str, metric_name: str) -> str: + metric_text = normalize_metric_text(metric_name) + source_text = normalize_metric_text(source_sheet) + if is_pts_test_run_times_metric(source_text, metric_text): + return PTS_TEST_RUN_TIMES_METRIC + base_metric = flattened_duplicate_base_metric(metric_text) + if ( + base_metric is not None + and target_metric_kind_for(source_sheet, logical_test, base_metric) is not None + ): + return base_metric + return metric_name + + +def is_pts_test_run_times_metric(source_text: str, metric_text: str) -> bool: + return source_text == "pts" and bool(PTS_TEST_RUN_TIMES_RE.fullmatch(metric_text)) + + +def flattened_duplicate_base_metric(metric_text: str) -> str | None: + if not is_flattened_duplicate_metric(metric_text): + return None + return FLATTENED_DUPLICATE_SUFFIX_RE.sub("", metric_text) + + +def is_flattened_duplicate_metric(metric_text: str) -> bool: + return bool(FLATTENED_DUPLICATE_SUFFIX_RE.search(metric_text)) + + +def has_technical_iperf_metric_part(metric_text: str) -> bool: + return any(part in metric_text for part in IPERF_TECHNICAL_METRIC_PARTS) + + +def is_iperf_cpu_host_metric(metric_text: str) -> bool: + return bool(IPERF_CPU_HOST_METRIC_RE.fullmatch(normalize_metric_text(metric_text))) + + +def normalized_iperf_metric_name(metric_text: str) -> str: + return IPERF_PREFIX_RE.sub("", metric_text) + + +def normalized_target_metric_name(metric_text: str) -> str: + metric_text = normalize_metric_text(metric_text) + return normalized_iperf_metric_name(metric_text) + + +def is_target_iperf_metric(metric_text: str) -> bool: + return bool(IPERF_TARGET_METRIC_RE.fullmatch(normalized_iperf_metric_name(metric_text))) + + +def is_iperf_metric_context(bucket: MetricBucket) -> bool: + return is_iperf_metric_context_text( + normalize_metric_text(bucket.source_sheet), + normalize_metric_text(bucket.logical_test), + normalize_metric_text(bucket.metric_name), + ) + + +def build_test_status_rows(rows: list[MetricStatusRow]) -> list[TestStatusRow]: + grouped: dict[tuple[int, str, str, str], list[MetricStatusRow]] = {} + for row in rows: + key = (row.group_order, row.group_label, row.source_sheet, row.test_name) + grouped.setdefault(key, []).append(row) + + result: list[TestStatusRow] = [] + for key in sorted(grouped): + group_rows = grouped[key] + failed_rows = [row for row in group_rows if row.status == STATUS_FAIL] + value_deltas = [ + row.value_delta_percent for row in group_rows if row.value_delta_percent is not None + ] + performance_deltas = [ + row.performance_delta_percent + for row in group_rows + if row.performance_delta_percent is not None + ] + result.append( + TestStatusRow( + status=STATUS_FAIL if failed_rows else STATUS_OK, + group_order=key[0], + group_label=key[1], + source_sheet=key[2], + test_name=key[3], + baseline_distro=BASELINE_DISTRO, + compared_distro=COMPARED_DISTRO, + epsilon_percent=group_rows[0].epsilon_percent, + metrics_total=len(group_rows), + passed_metrics=sum(1 for row in group_rows if row.status == STATUS_OK), + failed_metrics=len(failed_rows), + mean_value_delta_percent=(statistics.fmean(value_deltas) if value_deltas else None), + mean_performance_delta_percent=( + statistics.fmean(performance_deltas) if performance_deltas else None + ), + failed_metric_names=", ".join(row.metric_name for row in failed_rows), + ), + ) + return result + + +def mean_stats_or_none(values: list[float]) -> MeanStats | None: + if not values: + return None + return MeanStats( + mean=statistics.fmean(values), + sample_count=len(values), + ) + + +def metric_direction(bucket: MetricBucket) -> MetricDirection: + metric_text = normalize_metric_text(bucket.metric_name) + target_kind = target_metric_kind(bucket) + + if target_kind is None: + return MetricDirection( + better_direction=NOT_COMPARABLE_DIRECTION, + interpretation="not a target metric", + reason="the metric is not in the DB-recorded target metric allowlist", + comparable=False, + ) + + direction = inferred_metric_direction( + normalize_metric_text(bucket.source_sheet), + normalize_metric_text(bucket.logical_test), + metric_text, + ) + return MetricDirection( + better_direction=direction.better_direction, + interpretation=f"{target_kind}: {direction.interpretation}", + reason=direction.reason, + comparable=direction.comparable, + ) + + +def is_lower_is_better_target(metric_text: str) -> bool: + normalized_metric = normalized_target_metric_name(metric_text) + return ( + normalized_metric in LOWER_IS_BETTER_TARGETS + or bool(IPERF_LOWER_IS_BETTER_RE.search(normalized_metric)) + or bool(FIO_LOWER_IS_BETTER_RE.fullmatch(normalized_metric)) + ) + + +def flattened_duplicate_direction(metric_text: str) -> MetricDirection | None: + if is_flattened_duplicate_metric(metric_text): + return MetricDirection( + better_direction=NOT_COMPARABLE_DIRECTION, + interpretation="flattened duplicate column", + reason=( + "numeric suffixes such as _2, _13 or _1000 are produced by " + "flattening arrays and are not stable target metrics" + ), + comparable=False, + ) + return None + + +def flattened_iperf_direction(metric_text: str) -> MetricDirection | None: + if IPERF_INTERVAL_ANY_RE.search(metric_text): + return MetricDirection( + better_direction=NOT_COMPARABLE_DIRECTION, + interpretation="flattened iperf interval column", + reason=( + "iperf interval fields are intermediate time-series values, " + "not final PASS/FAIL metrics" + ), + comparable=False, + ) + return None + + +def iperf_target_direction(bucket: MetricBucket, metric_text: str) -> MetricDirection | None: + if not is_iperf_metric_context(bucket) or not is_target_iperf_metric(metric_text): + return None + + base_metric = normalized_iperf_metric_name(metric_text) + tokens = set(base_metric.split("_")) + if metric_has_any_pattern( + base_metric, + tokens, + { + "jitter", + "jitter_ms", + "lost", + "lost_packets", + "lost_percent", + "out_of_order", + "retransmit", + "retransmits", + "rtt", + }, + ): + return comparable_direction( + LOWER_IS_BETTER, + "iperf quality/error metric", + "lower values mean less loss, jitter, retransmits, RTT, or packet disorder", + ) + + return comparable_direction( + HIGHER_IS_BETTER, + "iperf throughput metric", + "higher values mean more network throughput or transferred data", + ) + + +def technical_metric_direction(metric_tokens: set[str]) -> MetricDirection | None: + if metric_tokens & TECHNICAL_OR_ID_PATTERNS: + return MetricDirection( + better_direction=NOT_COMPARABLE_DIRECTION, + interpretation="technical identifier or metadata", + reason="technical identifiers are not performance metrics", + comparable=False, + ) + return None + + +def explicit_metric_direction(text: str, tokens: set[str]) -> MetricDirection | None: + comparable = comparable_explicit_direction(text, tokens) + if comparable is not None: + return comparable + return non_comparable_explicit_direction(text, tokens) + + +def comparable_explicit_direction(text: str, tokens: set[str]) -> MetricDirection | None: + if metric_has_any_pattern(text, tokens, LOWER_IS_BETTER_PATTERNS): + return comparable_direction( + LOWER_IS_BETTER, + "error, loss, jitter, latency, or failure metric", + "lower values mean fewer errors, less loss, or less waiting", + ) + if metric_has_any_pattern(text, tokens, THROUGHPUT_PATTERNS | HIGHER_IS_BETTER_PATTERNS): + return comparable_direction( + HIGHER_IS_BETTER, + "throughput, operations, packets, requests, or success metric", + "higher values mean more work completed or more successful outcomes", + ) + if metric_has_any_pattern(text, tokens, TIME_PATTERNS): + return comparable_direction( + LOWER_IS_BETTER, + "time or duration metric", + "lower values mean the same work finished sooner", + ) + if metric_has_any_pattern(text, tokens, RESOURCE_USAGE_PATTERNS | MEMORY_PATTERNS): + return comparable_direction( + LOWER_IS_BETTER, + "resource usage metric", + "lower values mean less CPU, memory, or system resource consumption", + ) + return None + + +def non_comparable_explicit_direction(text: str, tokens: set[str]) -> MetricDirection | None: + if metric_has_any_pattern(text, tokens, PERCENT_PATTERNS): + return MetricDirection( + better_direction=NOT_COMPARABLE_DIRECTION, + interpretation="generic percentage metric", + reason="percentage metrics need domain context before higher/lower can be judged", + comparable=False, + ) + if metric_has_any_pattern(text, tokens, COUNT_PATTERNS): + return MetricDirection( + better_direction=NOT_COMPARABLE_DIRECTION, + interpretation="generic count metric", + reason="count metrics can be good or bad depending on what is counted", + comparable=False, + ) + return None + + +def generic_context_direction(text: str, tokens: set[str]) -> MetricDirection | None: + if is_known_throughput_test(text, tokens): + return comparable_direction( + HIGHER_IS_BETTER, + "generic value inside throughput benchmark", + "the surrounding test is a throughput benchmark, so higher is treated as better", + ) + if is_known_time_test(text, tokens): + return comparable_direction( + LOWER_IS_BETTER, + "generic value inside timing benchmark", + "the surrounding test is a timing benchmark, so lower is treated as better", + ) + return None + + +def comparable_direction( + better_direction: str, + interpretation: str, + reason: str, +) -> MetricDirection: + return MetricDirection( + better_direction=better_direction, + interpretation=interpretation, + reason=reason, + comparable=True, + ) + + +def is_known_throughput_test(text: str, tokens: set[str]) -> bool: + return metric_has_any_pattern( + text, + tokens, + IPERF_PATTERNS | STRESS_NG_PATTERNS | {"iperf3", "network", "network_loopback"}, + ) + + +def is_known_time_test(text: str, tokens: set[str]) -> bool: + return metric_has_any_pattern( + text, + tokens, + {"clock", "ctx_clock", "systemd", "systemd_analyze"}, + ) + + +def percent_delta(value: float, baseline: float) -> float | None: + if abs(baseline) <= NONZERO_EPSILON: + if abs(value) <= NONZERO_EPSILON: + return 0.0 + return None + return (value - baseline) / abs(baseline) * 100 + + +def directional_percent_delta( + value_delta_percent: float | None, + better_direction: str, +) -> float | None: + if value_delta_percent is None or better_direction == NOT_COMPARABLE_DIRECTION: + return None + if better_direction == LOWER_IS_BETTER: + return -value_delta_percent + return value_delta_percent + + +def performance_change_for_values( + baseline: float, + compared: float, + direction: MetricDirection, + epsilon_percent: float, +) -> str: + if not direction.comparable: + return CHANGE_NOT_COMPARABLE + + value_delta = percent_delta(compared, baseline) + performance_delta = directional_percent_delta(value_delta, direction.better_direction) + if performance_delta is not None: + return performance_change_for(performance_delta, direction, epsilon_percent) + + if abs(baseline) <= NONZERO_EPSILON: + if abs(compared) <= NONZERO_EPSILON: + return CHANGE_UNCHANGED + if direction.better_direction == LOWER_IS_BETTER: + return CHANGE_DEGRADED if compared > baseline else CHANGE_IMPROVED + return CHANGE_IMPROVED if compared > baseline else CHANGE_DEGRADED + + return CHANGE_NOT_COMPARABLE + + +def raw_value_change_for( # noqa: PLR0911 + value_delta_percent: float | None, + epsilon_percent: float, + *, + baseline: float | None = None, + compared: float | None = None, +) -> str: + if value_delta_percent is not None: + if abs(value_delta_percent) <= epsilon_percent: + return RAW_UNCHANGED + if value_delta_percent > 0: + return RAW_INCREASED + return RAW_DECREASED + if baseline is not None and compared is not None: + if abs(compared - baseline) <= NONZERO_EPSILON: + return RAW_UNCHANGED + if compared > baseline: + return RAW_INCREASED + return RAW_DECREASED + return RAW_UNAVAILABLE + + +def performance_change_for( + performance_delta_percent: float | None, + direction: MetricDirection, + epsilon_percent: float, +) -> str: + if not direction.comparable or performance_delta_percent is None: + return CHANGE_NOT_COMPARABLE + if abs(performance_delta_percent) <= epsilon_percent: + return CHANGE_UNCHANGED + if performance_delta_percent > 0: + return CHANGE_IMPROVED + return CHANGE_DEGRADED + + +def status_for(performance_change: str) -> str: + if performance_change in {CHANGE_IMPROVED, CHANGE_UNCHANGED}: + return STATUS_OK + if performance_change == CHANGE_DEGRADED: + return STATUS_FAIL + return STATUS_NOT_COMPARABLE + + +def write_status_workbook( + output_path: Path, + rows: list[MetricStatusRow], + groups: list[ComparisonGroup], + *, + epsilon_percent: float, +) -> None: + output_path.parent.mkdir(parents=True, exist_ok=True) + workbook = Workbook() + workbook.remove(workbook.active) + used_names: set[str] = set() + test_rows = build_test_status_rows(rows) + + write_summary_sheet( + workbook, + used_names, + rows, + test_rows, + groups, + epsilon_percent=epsilon_percent, + ) + write_test_status_sheet(workbook, used_names, test_rows) + write_metric_status_sheet(workbook, used_names, rows) + + workbook.save(output_path) + + +def write_summary_sheet( # noqa: PLR0913 + workbook: Workbook, + used_names: set[str], + rows: list[MetricStatusRow], + test_rows: list[TestStatusRow], + groups: list[ComparisonGroup], + *, + epsilon_percent: float, +) -> None: + metric_status_counts = count_by_status(rows) + test_status_counts = count_by_test_status(test_rows) + matrix: list[list[Any]] = [ + ["field", "value"], + ["baseline_distro", BASELINE_DISTRO], + ["compared_distro", COMPARED_DISTRO], + ["epsilon_percent", epsilon_percent], + ["comparison_groups", len(groups)], + ["test_rows", len(test_rows)], + ["metric_rows", len(rows)], + ["test_PASS", test_status_counts.get(STATUS_OK, 0)], + ["test_FAIL", test_status_counts.get(STATUS_FAIL, 0)], + ["metric_PASS", metric_status_counts.get(STATUS_OK, 0)], + ["metric_FAIL", metric_status_counts.get(STATUS_FAIL, 0)], + ] + ws = workbook.create_sheet(unique_sheet_name(SUMMARY_SHEET, used_names)) + write_matrix(ws, matrix) + style_table(ws, len(matrix), len(matrix[0])) + + +def count_by_status(rows: list[MetricStatusRow]) -> dict[str, int]: + result: dict[str, int] = {} + for row in rows: + result[row.status] = result.get(row.status, 0) + 1 + return result + + +def count_by_test_status(rows: list[TestStatusRow]) -> dict[str, int]: + result: dict[str, int] = {} + for row in rows: + result[row.status] = result.get(row.status, 0) + 1 + return result + + +def write_test_status_sheet( + workbook: Workbook, + used_names: set[str], + rows: list[TestStatusRow], +) -> None: + matrix = [test_status_headers()] + matrix.extend(test_status_row_values(row) for row in rows) + ws = workbook.create_sheet(unique_sheet_name(TEST_STATUS_SHEET, used_names)) + write_matrix(ws, matrix) + style_status_table(ws, len(matrix), len(matrix[0])) + ws.freeze_panes = "A2" + ws.auto_filter.ref = ws.dimensions + + +def test_status_headers() -> list[str]: + return [ + "status", + "group_order", + "group_label", + "source_sheet", + "test_name", + "baseline_distro", + "compared_distro", + "epsilon_percent", + "metrics_total", + "passed_metrics", + "failed_metrics", + "mean_value_delta_percent", + "mean_performance_delta_percent", + "failed_metric_names", + ] + + +def test_status_row_values(row: TestStatusRow) -> list[Any]: + return [ + row.status, + row.group_order, + row.group_label, + row.source_sheet, + row.test_name, + row.baseline_distro, + row.compared_distro, + row.epsilon_percent, + row.metrics_total, + row.passed_metrics, + row.failed_metrics, + row.mean_value_delta_percent, + row.mean_performance_delta_percent, + row.failed_metric_names, + ] + + +def write_metric_status_sheet( + workbook: Workbook, + used_names: set[str], + rows: list[MetricStatusRow], +) -> None: + matrix = [metric_status_headers()] + matrix.extend(metric_status_row_values(row) for row in rows) + ws = workbook.create_sheet(unique_sheet_name(METRIC_STATUS_SHEET, used_names)) + write_matrix(ws, matrix) + style_status_table(ws, len(matrix), len(matrix[0])) + ws.freeze_panes = "A2" + ws.auto_filter.ref = ws.dimensions + + +def metric_status_headers() -> list[str]: + return [ + "status", + "performance_change", + "raw_value_change", + "group_order", + "group_label", + "source_sheet", + "test_name", + "metric_name", + "baseline_distro", + "baseline_mean", + "baseline_sample_count", + "compared_distro", + "compared_mean", + "compared_sample_count", + "better_direction", + "epsilon_percent", + "value_delta_percent", + "performance_delta_percent", + ] + + +def metric_status_row_values(row: MetricStatusRow) -> list[Any]: + return [ + row.status, + row.performance_change, + row.raw_value_change, + row.group_order, + row.group_label, + row.source_sheet, + row.test_name, + row.metric_name, + row.baseline_distro, + row.baseline_mean, + row.baseline_sample_count, + row.compared_distro, + row.compared_mean, + row.compared_sample_count, + row.better_direction, + row.epsilon_percent, + row.value_delta_percent, + row.performance_delta_percent, + ] + + +def style_status_table(ws: Worksheet, row_count: int, col_count: int) -> None: + style_table(ws, row_count, col_count) + status_fills = { + STATUS_OK: PatternFill("solid", fgColor="D9EAD3"), + STATUS_FAIL: PatternFill("solid", fgColor="F4CCCC"), + STATUS_NOT_COMPARABLE: PatternFill("solid", fgColor="FFF2CC"), + } + for row_index in range(2, row_count + 1): + status_cell = ws.cell(row=row_index, column=1) + fill = status_fills.get(str(status_cell.value or "")) + if fill is not None: + status_cell.fill = fill + + for column_name in ( + "epsilon_percent", + "value_delta_percent", + "performance_delta_percent", + "mean_value_delta_percent", + "mean_performance_delta_percent", + ): + column_index_1based = worksheet_column_index(ws, column_name) + if column_index_1based is None: + continue + for row_index in range(2, row_count + 1): + ws.cell(row=row_index, column=column_index_1based).number_format = "0.00" + + +def worksheet_column_index(ws: Worksheet, column_name: str) -> int | None: + for cell in ws[1]: + if cell.value == column_name: + return cell.column + return None diff --git a/src/imgtests/reporting/excel_export.py b/src/imgtests/reporting/excel_export.py index cf535a23..99a937f3 100644 --- a/src/imgtests/reporting/excel_export.py +++ b/src/imgtests/reporting/excel_export.py @@ -1,6 +1,5 @@ from __future__ import annotations -import argparse import json import logging import shlex @@ -17,22 +16,31 @@ from imgtests.database.models.configuration import ConfigurationBase from imgtests.database.models.experiment import ExperimentBase from imgtests.database.models.util_run_result import UtilRunResult +from imgtests.reporting.cli import ( + DISTRO_COMPARISON_COMMAND, + DISTRO_COMPARISON_STATUS_COMMAND, + EXPORT_TABLES, + parse_args, +) from imgtests.reporting.distro_comparison_export import ( DistroComparisonExportOptions, - add_distro_comparison_arguments, export_distro_comparison_to_excel, ) +from imgtests.reporting.distro_comparison_status_export import ( + DistroComparisonStatusOptions, + export_distro_comparison_status_to_excel, +) from imgtests.types import Distro if TYPE_CHECKING: + import argparse + from sqlalchemy.engine import Engine -TABLES: Final = ("experiment", "util_run_result") logger = logging.getLogger(__name__) + CONFIGURATION_SHEET: Final = "configuration" DEFAULT_OUTPUT_FILENAME: Final = "report.xlsx" -DATABASE_COMMAND: Final = "database" -DISTRO_COMPARISON_COMMAND: Final = "distro-comparison" JSON_COLUMNS: Final = { "util_run_result": {"result"}, @@ -81,71 +89,15 @@ } -def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: - parser = argparse.ArgumentParser( - description="Export imgtests database data and report comparisons to XLSX workbooks.", - ) - subparsers = parser.add_subparsers(dest="command", required=True) - - database_parser = subparsers.add_parser( - DATABASE_COMMAND, - help="export database tables to an XLSX workbook", - ) - add_database_arguments(database_parser) - database_parser.set_defaults(command=DATABASE_COMMAND) - - distro_comparison_parser = subparsers.add_parser( - DISTRO_COMPARISON_COMMAND, - help="build Poky/SUSE comparison tables and charts from an exported report.xlsx", - ) - add_distro_comparison_arguments(distro_comparison_parser) - distro_comparison_parser.set_defaults(command=DISTRO_COMPARISON_COMMAND) - - return parser.parse_args(argv) - - -def add_database_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument( - "output", - type=Path, - help="Path to the output .xlsx file.", - ) - parser.add_argument( - "--db-url", - required=True, - help=("SQLAlchemy database URL."), - ) - parser.add_argument( - "--tables", - nargs="+", - choices=TABLES, - default=list(TABLES), - help="Tables to export. Defaults to all project result tables.", - ) - parser.add_argument( - "--configuration-id", - dest="configuration_ids", - action="append", - required=True, - metavar="DISTRO=ID", - type=parse_configuration_id_override, - help=( - "Configuration id to write for a distribution. " - "Must be passed for each supported distribution, for example: " - "--configuration-id poky=10 --configuration-id suse=20." - ), - ) - - def export_database_to_excel( engine: Engine, output_path: Path, configuration_ids: dict[str, int], - tables: Sequence[str] = tuple(TABLES), + tables: Sequence[str] = tuple(EXPORT_TABLES), ) -> Path: output_path = prepare_output_path(output_path) distributions = build_distribution_records(configuration_ids) - unsupported_tables = [table for table in tables if table not in TABLES] + unsupported_tables = [table for table in tables if table not in EXPORT_TABLES] if unsupported_tables: joined_tables = ", ".join(unsupported_tables) @@ -332,34 +284,6 @@ def build_distribution_records( } -def parse_configuration_id_override(value: str) -> tuple[str, int]: - distribution_name, separator, raw_id = value.partition("=") - - if not separator: - msg = "Expected configuration id in DISTRO=ID format." - raise argparse.ArgumentTypeError(msg) - - distribution_name = distribution_name.strip().lower() - raw_id = raw_id.strip() - - if distribution_name not in DISTRIBUTION_DESCRIPTIONS: - valid_names = ", ".join(DISTRIBUTION_DESCRIPTIONS) - msg = f"Unsupported distribution '{distribution_name}'. Available: {valid_names}." - raise argparse.ArgumentTypeError(msg) - - try: - configuration_id = int(raw_id) - except ValueError as exc: - msg = f"Configuration id for '{distribution_name}' must be an integer." - raise argparse.ArgumentTypeError(msg) from exc - - if configuration_id <= 0: - msg = f"Configuration id for '{distribution_name}' must be positive." - raise argparse.ArgumentTypeError(msg) - - return distribution_name, configuration_id - - def distribution_name_from_os(os_name: Any) -> str: value = str(os_name or "").strip().lower() @@ -856,6 +780,10 @@ def main() -> None: export_distro_comparison_command(args) return + if args.command == DISTRO_COMPARISON_STATUS_COMMAND: + export_distro_comparison_status_command(args) + return + engine = create_engine(args.db_url) try: @@ -877,7 +805,6 @@ def export_distro_comparison_command(args: argparse.Namespace) -> None: options=DistroComparisonExportOptions( output_path=args.output, experiment_ids=args.experiment_ids, - latest_pair_only=args.latest_pair_only, max_charts=args.max_charts, charts_per_sheet=args.charts_per_sheet, copy_source_sheets=args.copy_source_sheets, @@ -887,5 +814,17 @@ def export_distro_comparison_command(args: argparse.Namespace) -> None: logger.info("Exported comparison XLSX file: %s", output_path) +def export_distro_comparison_status_command(args: argparse.Namespace) -> None: + output_path = export_distro_comparison_status_to_excel( + input_path=args.input, + options=DistroComparisonStatusOptions( + output_path=args.output, + experiment_ids=args.experiment_ids, + epsilon_percent=args.epsilon_percent, + ), + ) + logger.info("Exported comparison status XLSX file: %s", output_path) + + if __name__ == "__main__": main() diff --git a/tests/unit/imgtests/reporting/__init__.py b/tests/unit/imgtests/reporting/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/imgtests/reporting/test_cli.py b/tests/unit/imgtests/reporting/test_cli.py new file mode 100644 index 00000000..5ac235fa --- /dev/null +++ b/tests/unit/imgtests/reporting/test_cli.py @@ -0,0 +1,68 @@ +import argparse + +import pytest + +from imgtests.constant import DISTRIBUTION_DESCRIPTIONS +from imgtests.reporting.cli import _parse_configuration_id_override + + +class TestParseConfigurationIdOverride: + @pytest.mark.parametrize( + ("value", "expected"), + [ + ("poky=10", ("poky", 10)), + ("suse=20", ("suse", 20)), + ("POKY=100", ("poky", 100)), + ("SUSE=200", ("suse", 200)), + ("poky = 10 ", ("poky", 10)), + ("suse = 20 ", ("suse", 20)), + ], + ids=[ + "Valid poky configuration id.", + "Valid suse configuration id.", + "Uppercase distribution name (poky).", + "Uppercase distribution name (suse).", + "Whitespace around equals sign and values.", + "Whitespace around equals sign and values.", + ], + ) + def test_valid_inputs(self, value: str, expected: tuple[str, int]) -> None: + assert _parse_configuration_id_override(value) == expected + + def test_invalid_format_missing_equals(self) -> None: + value = "poky10" + with pytest.raises(argparse.ArgumentTypeError) as exc_info: + _parse_configuration_id_override(value) + assert "Expected configuration id in DISTRO=ID format" in str(exc_info.value) + + def test_invalid_format_empty_distribution(self) -> None: + value = "=10" + with pytest.raises(argparse.ArgumentTypeError) as exc_info: + _parse_configuration_id_override(value) + assert "Unsupported distribution ''." in str(exc_info.value) + + def test_invalid_distribution_name(self) -> None: + value = "ubuntu=10" + with pytest.raises(argparse.ArgumentTypeError) as exc_info: + _parse_configuration_id_override(value) + assert "Unsupported distribution" in str(exc_info.value) + valid_names = ", ".join(DISTRIBUTION_DESCRIPTIONS) + assert valid_names in str(exc_info.value) + + def test_invalid_configuration_id_non_integer(self) -> None: + value = "poky=abc" + with pytest.raises(argparse.ArgumentTypeError) as exc_info: + _parse_configuration_id_override(value) + assert "Configuration id for 'poky' must be an integer" in str(exc_info.value) + + def test_invalid_configuration_id_zero(self) -> None: + value = "poky=0" + with pytest.raises(argparse.ArgumentTypeError) as exc_info: + _parse_configuration_id_override(value) + assert "Configuration id for 'poky' must be positive" in str(exc_info.value) + + def test_invalid_configuration_id_negative(self) -> None: + value = "poky=-5" + with pytest.raises(argparse.ArgumentTypeError) as exc_info: + _parse_configuration_id_override(value) + assert "Configuration id for 'poky' must be positive" in str(exc_info.value)