diff --git a/README.md b/README.md index e9a7059..880c198 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ local-ci-test-system/ ## Documentation -- **[User Guide](cli/Usage%20Guide.md)** — Installation, configuration, commands, troubleshooting +- **[User Guide](cli/Usage%20Guide.md)** — Installation, configuration, commands, troubleshooting (includes [workflow patch pipeline](cli/Usage%20Guide.md#workflow-patch-pipeline) and plugin registration) - **[Design Guide](docs/Design%20Guide.md)** — Architecture and caching - **[Performance and Caching](docs/Performance%20and%20Caching.md)** — Cache layout and config - **[Preparation and Plan](docs/Preparation%20and%20Plan.md)** — Implementation plan and issue breakdown diff --git a/cli/Usage Guide.md b/cli/Usage Guide.md index 8f2d31f..3b91533 100644 --- a/cli/Usage Guide.md +++ b/cli/Usage Guide.md @@ -16,6 +16,7 @@ - [Configuration](#configuration) - [Creating a Config File](#creating-a-config-file) - [Config File Reference](#config-file-reference) + - [Workflow patch pipeline](#workflow-patch-pipeline) - [Viewing and Editing Config](#viewing-and-editing-config) - [Commands](#commands) - [Global Options](#global-options) @@ -310,8 +311,105 @@ There is no CLI flag for `stop_on_first_failure`; set it in `.localci.yml` or wi | `priorities` | Override execution order (lower number runs first) | | `images` | Where Docker images are stored, auto-build, cleanup | | `cache` | ccache, Boost dependency, and CMake config caching | +| `patches` | Enable/disable workflow patch steps; project literals; plugin steps | +| `project` | Repository name and native image prefix for `act` command building | | `execution` | Timeouts, container cleanup, failure behaviour | +#### Workflow patch pipeline + +Before each job, Local CI copies your workflow YAML and applies a **patch +pipeline** (text-line transforms, not a YAML round-trip) so jobs run under +[act](https://github.com/nektos/act) with local images, cache bind mounts, and +other local-CI adaptations. + +**Built-in steps** (enable/disable each under `patches:`): + +| Step | Applies to | +|------|------------| +| `container_mounts` | Any workflow with a job `container:` block | +| `image_substitution` | Matrix jobs using a local Docker image | +| `codecov_skip` | Workflows that upload coverage via Codecov | +| `b2_source_cache`, `restore_capy_timestamps`, `capy_copy_preservation`, `b2_bootstrap_skip` | C++/B2 workflows whose shell/YAML literals match `patches.project` (Boost.Capy defaults) | + +**Project literals (`patches.project`)** — retarget the B2-oriented steps for +your workflow without forking Local CI. Defaults match Boost.Capy; override when +your workflow uses different directory names, step titles, or copy commands: + +```yaml +patches: + # Optional: turn off steps that do not apply + b2_source_cache: true + restore_capy_timestamps: true + capy_copy_preservation: true + b2_bootstrap_skip: true + + project: + boost_source_copy_command: "cp -rL dep-source dep-root" + boost_root_dir: dep-root + patch_dependency_step_name: "Patch Dependency" + project_source_dir: mylib-root + restore_timestamps_step_title: "Restore mylib source file timestamps" + file_stats_basename: .mylib-file-stats + workspace_libs_copy_marker: 'cp -r "$workspace_root"' + workspace_libs_copy_dest: "vendor/$pkg" + b2_workflow_action_marker: build-workflow + cached_module_libs_path: vendor/mylib +``` + +**Act event payload and images (`project`)** — override hard-coded Boost.Capy +assumptions for the simulated GitHub event and x86 image handling: + +```yaml +project: + repo_full_name: acme/my-cpp-lib + native_image_prefix: myproj- # x86 jobs skip linux/386 when image tag starts with this +``` + +**Custom patch steps (plugins)** — publish a step class and register it via the +`localci.patch_steps` entry-point group (see `localci.core.patch_registry`). +Enable it in `.localci.yml` with `patches.extra_steps`; enabled plugins append +after the built-in steps unless you set an explicit `patches.order`. + +Minimal plugin package: + +```python +# my_localci_patches/steps.py +from localci.core.patch_pipeline import PatchContext, PatchStep + +class MyPatchStep(PatchStep): + @property + def name(self) -> str: + return "my_patch" + + def apply(self, ctx: PatchContext) -> None: + ctx.lines.insert(0, "# patched by my_patch\n") +``` + +```toml +# pyproject.toml (install the package in the same environment as localci) +[project.entry-points."localci.patch_steps"] +my_patch = "my_localci_patches.steps:MyPatchStep" +``` + +```yaml +# .localci.yml +patches: + container_mounts: false + b2_source_cache: false + restore_capy_timestamps: false + capy_copy_preservation: false + b2_bootstrap_skip: false + image_substitution: false + codecov_skip: false + extra_steps: + my_patch: true + # Optional: explicit order (required if a plugin must run before a built-in step) + # order: + # - my_patch + # - container_mounts + # - image_substitution +``` + #### Build caching (Phase 2) When `cache.enabled` is true, Local CI bind-mounts host cache directories into diff --git a/cli/localci/core/command_builder.py b/cli/localci/core/command_builder.py index 5c4f032..fd92601 100644 --- a/cli/localci/core/command_builder.py +++ b/cli/localci/core/command_builder.py @@ -13,7 +13,12 @@ import tempfile from pathlib import Path -from localci.core.config import CacheConfig, ResolvedCachePaths +from localci.core.config import ( + DEFAULT_NATIVE_IMAGE_PREFIX, + DEFAULT_REPO_FULL_NAME, + CacheConfig, + ResolvedCachePaths, +) from localci.core.executor import ActCommand from localci.core.github_token import SENTINEL_GITHUB_TOKEN from localci.core.workflow import MatrixEntry @@ -52,7 +57,8 @@ def __init__( workflow_file: Path, project_dir: Path = Path("."), job_id: str = "build", - repo_full_name: str = "cppalliance/capy", + repo_full_name: str | None = None, + native_image_prefix: str | None = None, default_env: dict[str, str] | None = None, default_secrets: dict[str, str] | None = None, offline: bool = False, @@ -61,7 +67,8 @@ def __init__( self.workflow_file = workflow_file self.project_dir = project_dir self.job_id = job_id - self.repo_full_name = repo_full_name + self.repo_full_name = repo_full_name or DEFAULT_REPO_FULL_NAME + self.native_image_prefix = native_image_prefix or DEFAULT_NATIVE_IMAGE_PREFIX self.default_env = default_env or {} self.default_secrets = default_secrets or {} self.offline = offline @@ -183,11 +190,11 @@ def build( secrets.setdefault("GITHUB_TOKEN", SENTINEL_GITHUB_TOKEN) # Architecture: request linux/386 only when using a generic image - # (e.g. ubuntu:24.04). Our capy x86 image is amd64 with multilib, so - # we must not request 386 when using it or the daemon will reject. + # (e.g. ubuntu:24.04). Project-specific native images (e.g. capy x86) + # are amd64 with multilib, so we must not request 386 when using them. container_arch: str | None = None if entry.architecture == "x86" and ( - not image_tag or not str(image_tag).startswith("capy-") + not image_tag or not str(image_tag).startswith(self.native_image_prefix) ): container_arch = "linux/386" diff --git a/cli/localci/core/config.py b/cli/localci/core/config.py index bd1ede3..ce84fe7 100644 --- a/cli/localci/core/config.py +++ b/cli/localci/core/config.py @@ -184,7 +184,7 @@ class ExecutionConfig(BaseModel): stop_on_first_failure: bool = False -# Known workflow patch steps (order matches default pipeline sequence). +# Known built-in workflow patch steps (order matches default pipeline sequence). PATCH_STEP_NAMES: tuple[str, ...] = ( "container_mounts", "b2_source_cache", @@ -196,6 +196,24 @@ class ExecutionConfig(BaseModel): ) +class PatchProjectConfig(BaseModel): + """Project-specific literals for C++/Boost-oriented patch steps. + + Defaults preserve Boost.Capy workflow behaviour. Override for other projects. + """ + + boost_source_copy_command: str = "cp -rL boost-source boost-root" + boost_root_dir: str = "boost-root" + patch_dependency_step_name: str = "Patch Boost" + project_source_dir: str = "capy-root" + restore_timestamps_step_title: str = "Restore capy source file timestamps" + file_stats_basename: str = ".capy-file-stats" + workspace_libs_copy_marker: str = 'cp -r "$workspace_root"' + workspace_libs_copy_dest: str = "libs/$module" + b2_workflow_action_marker: str = "b2-workflow" + cached_module_libs_path: str = "libs/capy" + + class PatchesConfig(BaseModel): """Workflow patch pipeline settings (enable/disable individual patch types).""" @@ -207,6 +225,29 @@ class PatchesConfig(BaseModel): image_substitution: bool = True codecov_skip: bool = True order: list[str] | None = None + project: PatchProjectConfig = Field(default_factory=PatchProjectConfig) + extra_steps: dict[str, bool] = Field(default_factory=dict) + + @field_validator("extra_steps") + @classmethod + def validate_extra_steps(cls, v: dict[str, bool]) -> dict[str, bool]: + overlap = set(v.keys()) & set(PATCH_STEP_NAMES) + if overlap: + raise ValueError( + f"extra_steps must not duplicate built-in patch steps: " + f"{sorted(overlap)}" + ) + from localci.core.patch_registry import get_patch_step_registry + + builtin = set(PATCH_STEP_NAMES) + registered = set(get_patch_step_registry().keys()) - builtin + unknown = set(v.keys()) - registered + if unknown: + raise ValueError( + f"Unknown extra patch steps (register via localci.patch_steps " + f"entry points): {sorted(unknown)}" + ) + return v @field_validator("order") @classmethod @@ -219,7 +260,9 @@ def validate_order(cls, v: list[str] | None) -> list[str] | None: ) if len(v) != len(set(v)): raise ValueError("Duplicate step names in 'order'") - unknown = set(v) - set(PATCH_STEP_NAMES) + from localci.core.patch_registry import get_patch_step_registry + + unknown = set(v) - set(get_patch_step_registry().keys()) if unknown: raise ValueError(f"Unknown patch steps: {sorted(unknown)}") return v @@ -229,6 +272,7 @@ def validate_order_completeness(self) -> PatchesConfig: if self.order is None: return self enabled = {n for n in PATCH_STEP_NAMES if getattr(self, n, True)} + enabled |= {n for n, on in self.extra_steps.items() if on} missing = enabled - set(self.order) if missing: raise ValueError( @@ -236,6 +280,33 @@ def validate_order_completeness(self) -> PatchesConfig: ) return self + def is_step_enabled(self, name: str) -> bool: + """Return whether patch step *name* is enabled in this config.""" + if name in PATCH_STEP_NAMES: + return bool(getattr(self, name, True)) + return bool(self.extra_steps.get(name, False)) + + def resolved_order(self) -> list[str]: + """Return pipeline order: explicit ``order`` or built-ins + enabled plugins.""" + if self.order is not None: + return self.order + order = list(PATCH_STEP_NAMES) + order.extend( + name for name in sorted(self.extra_steps) if self.extra_steps[name] + ) + return order + + +DEFAULT_REPO_FULL_NAME = "cppalliance/capy" +DEFAULT_NATIVE_IMAGE_PREFIX = "capy-" + + +class ProjectConfig(BaseModel): + """Project identity and image conventions for act command building.""" + + repo_full_name: str = DEFAULT_REPO_FULL_NAME + native_image_prefix: str = DEFAULT_NATIVE_IMAGE_PREFIX + # --------------------------------------------------------------------------- # Root configuration model @@ -267,6 +338,7 @@ def expand_workflow(cls, v: Path) -> Path: logging: LoggingConfig = Field(default_factory=LoggingConfig) execution: ExecutionConfig = Field(default_factory=ExecutionConfig) patches: PatchesConfig = Field(default_factory=PatchesConfig) + project: ProjectConfig = Field(default_factory=ProjectConfig) # --------------------------------------------------------------------------- diff --git a/cli/localci/core/orchestrator.py b/cli/localci/core/orchestrator.py index 09ce0ac..026d301 100644 --- a/cli/localci/core/orchestrator.py +++ b/cli/localci/core/orchestrator.py @@ -20,7 +20,11 @@ from localci.core.cmake_cache import compute_cmake_input_digest from localci.core.command_builder import ActCommandBuilder -from localci.core.config import resolve_cache_paths +from localci.core.config import ( + DEFAULT_NATIVE_IMAGE_PREFIX, + DEFAULT_REPO_FULL_NAME, + resolve_cache_paths, +) from localci.core.executor import JobExecutor, JobResult, JobStatus from localci.core.models import JobEvent, JobEventType, QueuedJob from localci.core.queue import PriorityJobQueue @@ -65,6 +69,8 @@ class OrchestratorConfig: dispatch_interval: float = 0.1 default_secrets: dict[str, str] | None = None default_env: dict[str, str] | None = None + repo_full_name: str = DEFAULT_REPO_FULL_NAME + native_image_prefix: str = DEFAULT_NATIVE_IMAGE_PREFIX image_registry_path: Path | None = None verbose: bool = False offline: bool = False @@ -80,6 +86,7 @@ def from_config(cls, config: LocalCIConfig) -> OrchestratorConfig: disk_gb = float(getattr(rl, "disk_min_free_gb", 10.0)) images = getattr(config, "images", None) registry_path = getattr(images, "registry", None) if images else None + project = getattr(config, "project", None) return cls( max_parallel=getattr(config.parallel, "max_jobs", 8), cpu_threshold=float(cpu), @@ -90,6 +97,10 @@ def from_config(cls, config: LocalCIConfig) -> OrchestratorConfig: stop_on_first_failure=getattr( config.execution, "stop_on_first_failure", False ), + repo_full_name=getattr(project, "repo_full_name", DEFAULT_REPO_FULL_NAME), + native_image_prefix=getattr( + project, "native_image_prefix", DEFAULT_NATIVE_IMAGE_PREFIX + ), image_registry_path=registry_path, auto_build=getattr(images, "auto_build", True) if images else True, ) @@ -434,6 +445,8 @@ def _execute_job(self, job: QueuedJob) -> JobResult: workflow_file=self.workflow_file, project_dir=self.project_dir, job_id=job.job_id, + repo_full_name=self.config.repo_full_name, + native_image_prefix=self.config.native_image_prefix, default_secrets=self.config.default_secrets or {}, default_env=self.config.default_env or {}, offline=self.config.offline, diff --git a/cli/localci/core/patch_pipeline.py b/cli/localci/core/patch_pipeline.py index 886b296..919dd75 100644 --- a/cli/localci/core/patch_pipeline.py +++ b/cli/localci/core/patch_pipeline.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING -from localci.core.config import PATCH_STEP_NAMES, LocalCIConfig +from localci.core.config import LocalCIConfig logger = logging.getLogger(__name__) @@ -66,19 +66,20 @@ def steps(self) -> list[PatchStep]: @classmethod def from_config(cls, config: LocalCIConfig) -> PatchPipeline: """Build a pipeline from ``.localci.yml`` patch settings.""" - from localci.core.patch_steps import PATCH_STEP_REGISTRY + from localci.core.patch_registry import get_patch_step_registry patches = config.patches - order = patches.order or list(PATCH_STEP_NAMES) + order = patches.resolved_order() + registry = get_patch_step_registry() steps: list[PatchStep] = [] for name in order: - if not getattr(patches, name, True): + if not patches.is_step_enabled(name): continue - step_cls = PATCH_STEP_REGISTRY.get(name) + step_cls = registry.get(name) if step_cls is None: raise ValueError( f"Patch step {name!r} is enabled but not registered; " - "check PATCH_STEP_NAMES / PATCH_STEP_REGISTRY alignment" + "check PATCH_STEP_NAMES / patch step registry alignment" ) steps.append(step_cls()) return cls(steps) diff --git a/cli/localci/core/patch_registry.py b/cli/localci/core/patch_registry.py new file mode 100644 index 0000000..b3d541e --- /dev/null +++ b/cli/localci/core/patch_registry.py @@ -0,0 +1,50 @@ +"""Built-in and entry-point patch step registration. + +Built-in steps live in :data:`localci.core.patch_steps.PATCH_STEP_REGISTRY`. +Additional steps register via the ``localci.patch_steps`` entry-point group:: + + [project.entry-points."localci.patch_steps"] + my_step = "my_package.patch_steps:MyCustomStep" +""" + +from __future__ import annotations + +import logging +from functools import lru_cache +from importlib.metadata import entry_points +from typing import TYPE_CHECKING + +from localci.core.config import PATCH_STEP_NAMES + +logger = logging.getLogger(__name__) + +if TYPE_CHECKING: + from localci.core.patch_pipeline import PatchStep + + +@lru_cache(maxsize=1) +def get_patch_step_registry() -> dict[str, type[PatchStep]]: + """Return built-in patch steps merged with entry-point plugins.""" + from localci.core.patch_steps import PATCH_STEP_REGISTRY + + registry: dict[str, type[PatchStep]] = dict(PATCH_STEP_REGISTRY) + for ep in entry_points(group="localci.patch_steps"): + if ep.name in registry: + continue + try: + registry[ep.name] = ep.load() + except Exception: + logger.exception( + "Failed to load localci.patch_steps entry point %r", ep.name + ) + return registry + + +def clear_patch_step_registry_cache() -> None: + """Clear the registry cache (for tests that register plugins at runtime).""" + get_patch_step_registry.cache_clear() + + +def is_builtin_patch_step(name: str) -> bool: + """Return True when *name* is a built-in (non-plugin) patch step.""" + return name in PATCH_STEP_NAMES diff --git a/cli/localci/core/patch_steps.py b/cli/localci/core/patch_steps.py index c8dd2e3..e10a13c 100644 --- a/cli/localci/core/patch_steps.py +++ b/cli/localci/core/patch_steps.py @@ -4,6 +4,7 @@ import re +from localci.core.config import PatchProjectConfig from localci.core.patch_pipeline import PatchContext, PatchStep @@ -32,6 +33,10 @@ def _step_insert_indents(lines: list[str], step_start: int) -> tuple[str, str, s return list_indent, prop_indent, prop_indent + " " +def _project_settings(ctx: PatchContext) -> PatchProjectConfig: + return ctx.config.patches.project + + class ContainerMountsStep(PatchStep): """Inject cache bind-mount flags into the job container options.""" @@ -85,44 +90,52 @@ def name(self) -> str: return "b2_source_cache" def apply(self, ctx: PatchContext) -> None: + proj = _project_settings(ctx) + copy_cmd = proj.boost_source_copy_command + boost_root = proj.boost_root_dir + cached_libs = proj.cached_module_libs_path for i, line in enumerate(ctx.lines): - if ( - "cp -rL boost-source boost-root" in line - and "LOCALCI_B2_SOURCE_DIR" not in line - ): + if copy_cmd in line and "LOCALCI_B2_SOURCE_DIR" not in line: ind = line[: len(line) - len(line.lstrip())] ctx.lines[i] = ( f'{ind}if [ -n "${{LOCALCI_B2_SOURCE_DIR:-}}" ] && [ -f "${{LOCALCI_B2_SOURCE_DIR}}/Jamroot" ]; then\n' f"{ind} # Cache hit: leave headers untouched (stable timestamps) so b2 builds incrementally\n" - f'{ind} rm -rf "${{LOCALCI_B2_SOURCE_DIR}}/libs/capy" 2>/dev/null || true\n' - f'{ind} ln -sfn "${{LOCALCI_B2_SOURCE_DIR}}" boost-root\n' + f'{ind} rm -rf "${{LOCALCI_B2_SOURCE_DIR}}/{cached_libs}" 2>/dev/null || true\n' + f'{ind} ln -sfn "${{LOCALCI_B2_SOURCE_DIR}}" {boost_root}\n' f"{ind}else\n" - f"{ind} cp -rL boost-source boost-root\n" + f"{ind} {copy_cmd}\n" f'{ind} if [ -n "${{LOCALCI_B2_SOURCE_DIR:-}}" ]; then\n' f'{ind} mkdir -p "${{LOCALCI_B2_SOURCE_DIR}}"\n' - f'{ind} cp -a boost-root/. "${{LOCALCI_B2_SOURCE_DIR}}/"\n' + f'{ind} cp -a {boost_root}/. "${{LOCALCI_B2_SOURCE_DIR}}/"\n' f"{ind} fi\n" f"{ind}fi\n" ) return - self._skip("no cp -rL boost-source boost-root line found") + self._skip(f"no {copy_cmd!r} line found") return class RestoreCapyTimestampsStep(PatchStep): - """Inject restore-timestamps step before Patch Boost.""" + """Inject restore-timestamps step before the dependency patch step.""" @property def name(self) -> str: return "restore_capy_timestamps" def apply(self, ctx: PatchContext) -> None: + proj = _project_settings(ctx) + step_name = proj.patch_dependency_step_name + source_dir = proj.project_source_dir + stats_file = proj.file_stats_basename for i, line in enumerate(ctx.lines): - step_match = re.match(r"^(\s+)-\s+name:\s+Patch Boost", line) + step_match = re.match( + rf"^(\s+)-\s+name:\s+{re.escape(step_name)}", + line, + ) if not step_match: continue already_patched = any( - "capy-file-stats" in ctx.lines[j] for j in range(max(0, i - 15), i) + stats_file in ctx.lines[j] for j in range(max(0, i - 15), i) ) if already_patched: self._skip("restore step already present") @@ -131,48 +144,53 @@ def apply(self, ctx: PatchContext) -> None: prop_indent = list_indent + " " body_indent = prop_indent + " " new_step = [ - f"{list_indent}- name: Restore capy source file timestamps\n", + f"{list_indent}- name: {proj.restore_timestamps_step_title}\n", f"{prop_indent}run: |\n", - f'{body_indent}if [ -n "${{LOCALCI_B2_SOURCE_DIR:-}}" ] && [ -f "${{LOCALCI_B2_SOURCE_DIR}}/.capy-file-stats" ]; then\n', + f'{body_indent}if [ -n "${{LOCALCI_B2_SOURCE_DIR:-}}" ] && [ -f "${{LOCALCI_B2_SOURCE_DIR}}/{stats_file}" ]; then\n', f"{body_indent} while IFS=' ' read -r saved_mtime fhash relpath; do\n", - f'{body_indent} [ -f "capy-root/$relpath" ] || continue\n', - f"{body_indent} curr=$(sha256sum \"capy-root/$relpath\" 2>/dev/null | cut -d' ' -f1)\n", - f'{body_indent} [ "$curr" = "$fhash" ] && touch -d "@$saved_mtime" "capy-root/$relpath" 2>/dev/null || true\n', - f'{body_indent} done < "${{LOCALCI_B2_SOURCE_DIR}}/.capy-file-stats"\n', + f'{body_indent} [ -f "{source_dir}/$relpath" ] || continue\n', + f"{body_indent} curr=$(sha256sum \"{source_dir}/$relpath\" 2>/dev/null | cut -d' ' -f1)\n", + f'{body_indent} [ "$curr" = "$fhash" ] && touch -d "@$saved_mtime" "{source_dir}/$relpath" 2>/dev/null || true\n', + f'{body_indent} done < "${{LOCALCI_B2_SOURCE_DIR}}/{stats_file}"\n', f"{body_indent}fi\n", ] for j, new_line in enumerate(new_step): ctx.lines.insert(i + j, new_line) return - self._skip("Patch Boost step not found") + self._skip(f"{step_name!r} step not found") return class CapyCopyPreservationStep(PatchStep): - """Use cp -rp and save capy file stats for incremental b2 builds.""" + """Use cp -rp and save project file stats for incremental b2 builds.""" @property def name(self) -> str: return "capy_copy_preservation" def apply(self, ctx: PatchContext) -> None: + proj = _project_settings(ctx) + marker = proj.workspace_libs_copy_marker + dest = proj.workspace_libs_copy_dest + source_dir = proj.project_source_dir + stats_file = proj.file_stats_basename for i, line in enumerate(ctx.lines): - if 'cp -r "$workspace_root"' in line and "libs/" in line: + if marker in line and dest in line: ind = line[: len(line) - len(line.lstrip())] ctx.lines[i] = ( - f'{ind}cp -rp "$workspace_root"/capy-root "libs/$module"\n' + f'{ind}cp -rp "$workspace_root"/{source_dir} "{dest}"\n' f'{ind}if [ -n "${{LOCALCI_B2_SOURCE_DIR:-}}" ]; then\n' f'{ind} mkdir -p "${{LOCALCI_B2_SOURCE_DIR}}"\n' - f'{ind} find "$workspace_root/capy-root" -type f \\( -name "*.cpp" -o -name "*.hpp" -o -name "*.h" -o -name "*.ipp" \\) |\n' + f'{ind} find "$workspace_root/{source_dir}" -type f \\( -name "*.cpp" -o -name "*.hpp" -o -name "*.h" -o -name "*.ipp" \\) |\n' f"{ind} while IFS= read -r f; do\n" f'{ind} mtime=$(stat -c "%Y" "$f")\n' f'{ind} fhash=$(sha256sum "$f" | cut -d" " -f1)\n' - f'{ind} echo "$mtime $fhash ${{f#$workspace_root/capy-root/}}"\n' - f'{ind} done > "${{LOCALCI_B2_SOURCE_DIR}}/.capy-file-stats"\n' + f'{ind} echo "$mtime $fhash ${{f#$workspace_root/{source_dir}/}}"\n' + f'{ind} done > "${{LOCALCI_B2_SOURCE_DIR}}/{stats_file}"\n' f"{ind}fi\n" ) return - self._skip('no cp -r "$workspace_root" capy copy line found') + self._skip(f"no {marker!r} workspace copy line found") return @@ -184,8 +202,11 @@ def name(self) -> str: return "b2_bootstrap_skip" def apply(self, ctx: PatchContext) -> None: + proj = _project_settings(ctx) + action_marker = proj.b2_workflow_action_marker + boost_root = proj.boost_root_dir for i, line in enumerate(ctx.lines): - if "b2-workflow" in line and "uses:" in line: + if action_marker in line and "uses:" in line: step_start = i while step_start > 0: if re.match(r"^\s+-\s+name:\s*", ctx.lines[step_start]): @@ -205,14 +226,14 @@ def apply(self, ctx: PatchContext) -> None: f"{list_indent}- name: Skip b2 bootstrap (b2 binary cached)\n", f"{prop_indent}run: |\n", f'{body_indent}if [ -n "${{LOCALCI_B2_SOURCE_DIR:-}}" ] && [ -f "${{LOCALCI_B2_SOURCE_DIR}}/b2" ]; then\n', - f"{body_indent} printf '#!/bin/sh\\necho \"b2 binary cached, skipping bootstrap.\"\\n' > boost-root/bootstrap.sh\n", - f"{body_indent} chmod +x boost-root/bootstrap.sh\n", + f"{body_indent} printf '#!/bin/sh\\necho \"b2 binary cached, skipping bootstrap.\"\\n' > {boost_root}/bootstrap.sh\n", + f"{body_indent} chmod +x {boost_root}/bootstrap.sh\n", f"{body_indent}fi\n", ] for j, new_line in enumerate(new_step): ctx.lines.insert(step_start + j, new_line) return - self._skip("no b2-workflow uses step found") + self._skip(f"no {action_marker!r} uses step found") return diff --git a/cli/tests/fixtures/patcher/generic_cpp.yml b/cli/tests/fixtures/patcher/generic_cpp.yml new file mode 100644 index 0000000..17bcf11 --- /dev/null +++ b/cli/tests/fixtures/patcher/generic_cpp.yml @@ -0,0 +1,20 @@ +name: Generic C++ project +on: push +jobs: + build: + runs-on: ubuntu-latest + container: + image: ubuntu:24.04 + strategy: + matrix: + include: + - name: "build (ubuntu-24.04, gcc-15)" + runs-on: ubuntu-latest + container: ubuntu:24.04 + steps: + - name: Patch Dependency + run: | + cp -rL dep-source dep-root + cp -r "$workspace_root"/mylib-root "vendor/$pkg" + - name: Build Workflow + uses: org/cpp-actions/build-workflow@v1.0.0 diff --git a/cli/tests/test_executor.py b/cli/tests/test_executor.py index a7cf3b2..f130eec 100644 --- a/cli/tests/test_executor.py +++ b/cli/tests/test_executor.py @@ -948,6 +948,35 @@ def test_x86_architecture(self, tmp_path): assert cmd.container_architecture == "linux/386" + def test_x86_with_native_image_prefix_skips_386(self, tmp_path): + wf = tmp_path / "ci.yml" + wf.write_text("name: CI") + + builder = ActCommandBuilder( + workflow_file=wf, + native_image_prefix="myproj-", + ) + entry = _make_entry(architecture="x86") + cmd = builder.build(entry, image_tag="myproj-ubuntu-24.04-gcc15:latest") + + assert cmd.container_architecture is None + + def test_custom_repo_full_name_in_event(self, tmp_path): + wf = tmp_path / "ci.yml" + wf.write_text("name: CI") + + builder = ActCommandBuilder( + workflow_file=wf, + repo_full_name="acme/widget", + ) + entry = _make_entry() + cmd = builder.build(entry) + + assert cmd.event_file is not None + content = json.loads(cmd.event_file.read_text()) + assert content["push"]["repository"]["full_name"] == "acme/widget" + cmd.event_file.unlink() + def test_x86_64_no_arch_flag(self, tmp_path): wf = tmp_path / "ci.yml" wf.write_text("name: CI") diff --git a/cli/tests/test_patch_pipeline.py b/cli/tests/test_patch_pipeline.py index f45d0ed..7555421 100644 --- a/cli/tests/test_patch_pipeline.py +++ b/cli/tests/test_patch_pipeline.py @@ -8,8 +8,8 @@ import pytest from localci.cli.run.patcher import _write_patched_workflow -from localci.core.config import LocalCIConfig, PatchesConfig -from localci.core.patch_pipeline import PatchContext, PatchPipeline +from localci.core.config import LocalCIConfig, PatchesConfig, PatchProjectConfig +from localci.core.patch_pipeline import PatchContext, PatchPipeline, PatchStep from localci.core.patch_steps import ContainerMountsStep from localci.core.workflow import ( BuildSystem, @@ -24,6 +24,26 @@ FIXTURES_DIR = Path(__file__).parent / "fixtures" PIPELINE_WORKFLOW = FIXTURES_DIR / "patcher" / "pipeline_minimal.yml" +GENERIC_CPP_WORKFLOW = FIXTURES_DIR / "patcher" / "generic_cpp.yml" + + +def _generic_project_config() -> LocalCIConfig: + return LocalCIConfig( + patches=PatchesConfig( + project=PatchProjectConfig( + boost_source_copy_command="cp -rL dep-source dep-root", + boost_root_dir="dep-root", + patch_dependency_step_name="Patch Dependency", + project_source_dir="mylib-root", + restore_timestamps_step_title="Restore mylib source file timestamps", + file_stats_basename=".mylib-file-stats", + workspace_libs_copy_marker='cp -r "$workspace_root"', + workspace_libs_copy_dest="vendor/$pkg", + b2_workflow_action_marker="build-workflow", + cached_module_libs_path="vendor/mylib", + ) + ) + ) @pytest.fixture @@ -176,7 +196,10 @@ def test_from_config_raises_when_step_missing_from_registry( for name, cls in PATCH_STEP_REGISTRY.items() if name != "b2_source_cache" } - monkeypatch.setattr("localci.core.patch_steps.PATCH_STEP_REGISTRY", incomplete) + monkeypatch.setattr( + "localci.core.patch_registry.get_patch_step_registry", + lambda: incomplete, + ) with pytest.raises(ValueError, match="b2_source_cache"): PatchPipeline.from_config(LocalCIConfig()) @@ -214,3 +237,117 @@ def test_patch_step_skip_emits_warning(sample_entry: MatrixEntry, caplog) -> Non and "job_id and container_mount_options are required" in r.message for r in caplog.records ) + + +def test_generic_cpp_workflow_patches_with_project_config( + sample_entry: MatrixEntry, +) -> None: + """Non-Boost workflow patches through configurable project literals.""" + assert GENERIC_CPP_WORKFLOW.is_file() + patched = _write_patched_workflow( + GENERIC_CPP_WORKFLOW, + sample_entry, + config=_generic_project_config(), + ) + try: + content = patched.read_text() + assert "LOCALCI_B2_SOURCE_DIR" in content + assert "Restore mylib source file timestamps" in content + assert ".mylib-file-stats" in content + assert 'cp -rp "$workspace_root"/mylib-root "vendor/$pkg"' in content + assert "Skip b2 bootstrap" in content + assert "dep-root" in content + assert "boost-source" not in content + assert "capy-root" not in content + finally: + patched.unlink(missing_ok=True) + + +class _MarkerPluginStep(PatchStep): + """Test-only plugin step that prepends a marker comment.""" + + @property + def name(self) -> str: + return "marker_plugin" + + def apply(self, ctx: PatchContext) -> None: + ctx.lines.insert(0, "# patched by marker_plugin\n") + + +def test_custom_plugin_step_via_registry( + workflow_path: Path, + sample_entry: MatrixEntry, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Entry-point-registered steps can be enabled via extra_steps.""" + from localci.core.patch_steps import PATCH_STEP_REGISTRY + + registry = dict(PATCH_STEP_REGISTRY) + registry["marker_plugin"] = _MarkerPluginStep + monkeypatch.setattr( + "localci.core.patch_registry.get_patch_step_registry", + lambda: registry, + ) + + cfg = LocalCIConfig( + patches=PatchesConfig( + container_mounts=False, + b2_source_cache=False, + restore_capy_timestamps=False, + capy_copy_preservation=False, + b2_bootstrap_skip=False, + image_substitution=False, + codecov_skip=False, + extra_steps={"marker_plugin": True}, + order=["marker_plugin"], + ) + ) + patched = _write_patched_workflow(workflow_path, sample_entry, config=cfg) + try: + assert patched.read_text().startswith("# patched by marker_plugin\n") + finally: + patched.unlink(missing_ok=True) + + +def test_custom_plugin_step_runs_without_explicit_order( + workflow_path: Path, + sample_entry: MatrixEntry, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Enabled extra_steps append to the default built-in order when order is omitted.""" + from localci.core.patch_steps import PATCH_STEP_REGISTRY + + registry = dict(PATCH_STEP_REGISTRY) + registry["marker_plugin"] = _MarkerPluginStep + monkeypatch.setattr( + "localci.core.patch_registry.get_patch_step_registry", + lambda: registry, + ) + + cfg = LocalCIConfig( + patches=PatchesConfig( + container_mounts=False, + b2_source_cache=False, + restore_capy_timestamps=False, + capy_copy_preservation=False, + b2_bootstrap_skip=False, + image_substitution=False, + codecov_skip=False, + extra_steps={"marker_plugin": True}, + ) + ) + patched = _write_patched_workflow(workflow_path, sample_entry, config=cfg) + try: + assert patched.read_text().startswith("# patched by marker_plugin\n") + finally: + patched.unlink(missing_ok=True) + + +def test_extra_steps_rejects_unknown_plugin() -> None: + with pytest.raises(ValueError, match="Unknown extra patch steps"): + PatchesConfig(extra_steps={"not_registered": True}) + + +def test_extra_steps_rejects_builtin_duplicate() -> None: + with pytest.raises(ValueError, match="must not duplicate built-in"): + PatchesConfig(extra_steps={"codecov_skip": True}) diff --git a/cli/tests/test_patcher_patches.py b/cli/tests/test_patcher_patches.py index 8b126ec..87d258e 100644 --- a/cli/tests/test_patcher_patches.py +++ b/cli/tests/test_patcher_patches.py @@ -326,7 +326,7 @@ def test_negative_leaves_workflow_unchanged_without_boost_copy_line( assert "cp -rL boost-source boost-root" not in content assert content == original.replace("\r\n", "\n") _assert_skip_warning( - caplog, "b2_source_cache", "no cp -rL boost-source boost-root line found" + caplog, "b2_source_cache", "'cp -rL boost-source boost-root' line found" ) @@ -366,7 +366,7 @@ def test_negative_skips_when_patch_boost_step_absent(self, patcher_paths, caplog content = _assert_valid_yaml(patched) assert "Restore capy source file timestamps" not in content _assert_skip_warning( - caplog, "restore_capy_timestamps", "Patch Boost step not found" + caplog, "restore_capy_timestamps", "'Patch Boost' step not found" ) @@ -399,7 +399,7 @@ def test_negative_no_cp_rp_injected_when_capy_copy_line_absent( _assert_skip_warning( caplog, "capy_copy_preservation", - 'no cp -r "$workspace_root" capy copy line found', + "no 'cp -r \"$workspace_root\"' workspace copy line found", ) @@ -425,7 +425,7 @@ def test_negative_skips_when_b2_workflow_step_absent(self, patcher_paths, caplog content = _assert_valid_yaml(patched) assert "Skip b2 bootstrap (b2 binary cached)" not in content _assert_skip_warning( - caplog, "b2_bootstrap_skip", "no b2-workflow uses step found" + caplog, "b2_bootstrap_skip", "'b2-workflow' uses step found" )