diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index 254a1f962..14fcdc662 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -220,6 +220,25 @@ jobs: uv run ./scripts/test-integration.sh timeout-minutes: 30 + - name: Run live generic GitLab smoke + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + env: + APM_E2E_TESTS: "1" + APM_RUN_INTEGRATION_TESTS: "1" + # Unset = collection-time skip. Set both vars after the public GitLab + # fixture exists; EXPECTED_SHA pins the fixture commit the smoke proves. + APM_LIVE_GENERIC_PACKAGE: ${{ vars.APM_LIVE_GENERIC_PACKAGE }} + APM_LIVE_GENERIC_EXPECTED_SHA: ${{ vars.APM_LIVE_GENERIC_EXPECTED_SHA }} + run: | + uv run pytest tests/integration/test_live_generic_gitlab_install.py \ + -m live_generic \ + -o addopts='' \ + -v \ + --tb=short + # The test's install subprocess timeout is 240s; keep the step + # larger so pytest can report fixture, network, or lockfile failures. + timeout-minutes: 10 + # ── PHASE 3: RELEASE VALIDATION (tag/schedule/dispatch only) ── - name: Prepare isolated release-validation environment if: github.ref_type == 'tag' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e2110e95..90b29a57c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Live GitLab install smoke-test infrastructure for `apm install + gitlab.com//`, gated by `APM_LIVE_GENERIC_PACKAGE` and pinned by + `APM_LIVE_GENERIC_EXPECTED_SHA` for scheduled/manual CI activation. (#1663, + tracks #1229) + ## [0.21.0] - 2026-06-19 ### Added diff --git a/docs/src/content/docs/contributing/integration-testing.md b/docs/src/content/docs/contributing/integration-testing.md index 1bb6d6cce..bfa44df1e 100644 --- a/docs/src/content/docs/contributing/integration-testing.md +++ b/docs/src/content/docs/contributing/integration-testing.md @@ -53,10 +53,12 @@ what the test family you want actually requires. | `requires_ado_pat` | Azure DevOps PAT for ADO host tests | `export ADO_APM_PAT=...` | | `requires_ado_bearer` | Azure CLI signed in + opt-in flag | `az login` and `export APM_TEST_ADO_BEARER=1` | | `requires_apm_binary` | A built `apm` binary on disk or `PATH` | `scripts/build-binary.sh` (or set `APM_BINARY_PATH`) | +| `requires_live_generic_fixture` | A live non-GitHub/non-ADO git fixture package is configured | `export APM_LIVE_GENERIC_PACKAGE=gitlab.com//` and `export APM_LIVE_GENERIC_EXPECTED_SHA=<40-char-sha>` | | `requires_runtime_codex` | The `codex` runtime installed under `~/.apm/runtimes/` | `apm runtime setup codex` | | `requires_runtime_copilot` | The GitHub Copilot CLI runtime installed under `~/.apm/runtimes/` | `apm runtime setup copilot` | | `requires_runtime_llm` | The `llm` runtime installed under `~/.apm/runtimes/` | `apm runtime setup llm` | -| `live` | Tests that hit real GitHub repos via cloning; deselected by default | Override the deselect: `pytest -m live tests/integration -v` | +| `live` | Tests that hit real remote repos via live network cloning; deselected by default | Override the deselect: `pytest -m live tests/integration -v` | +| `live_generic` | Live smoke tests for non-GitHub/non-ADO git-host install paths; deselected by default | Set the fixture env vars, then run `pytest -m live_generic tests/integration -v` | Without any of those env vars or runtimes a `pytest tests/integration` invocation is silent rather than red: every test is collected and @@ -74,8 +76,20 @@ uv run pytest tests/integration/test_golden_scenario_e2e.py -v # Run only a marker family uv run pytest tests/integration -m requires_github_token -v + +# Run the live GitLab smoke after maintainers publish and pin the fixture +export APM_RUN_INTEGRATION_TESTS=1 +export APM_LIVE_GENERIC_PACKAGE=gitlab.com// +export APM_LIVE_GENERIC_EXPECTED_SHA=<40-char-sha> +uv run pytest tests/integration/test_live_generic_gitlab_install.py -m live_generic -o addopts='' -v ``` +`APM_LIVE_GENERIC_HOST` is optional and defaults to `gitlab.com`. Set it only +when validating the same live-generic fixture flow against another GitLab host. +`APM_LIVE_GENERIC_EXPECTED_SHA` is required whenever +`APM_LIVE_GENERIC_PACKAGE` is set, so the smoke test proves a known fixture +commit rather than trusting whatever the remote default branch currently serves. + ### Apm binary resolution Tests that need to shell out to a real `apm` binary use the diff --git a/pyproject.toml b/pyproject.toml index 5124e5783..05912f109 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -152,10 +152,11 @@ exclude_lines = [ ] [tool.pytest.ini_options] -addopts = "-m 'not benchmark and not live'" +addopts = "-m 'not benchmark and not live and not live_generic'" markers = [ "integration: marks tests as integration tests that may require network access", - "live: marks tests that hit real GitHub repos (requires network + optional GITHUB_TOKEN)", + "live: marks tests that hit real remote repos via live network cloning", + "live_generic: marks tests that hit a live non-GitHub/non-ADO git host fixture", "slow: marks tests as slow running tests", "benchmark: marks performance benchmark tests (deselected by default, run with -m benchmark)", "requires_e2e_mode: requires APM_E2E_TESTS=1", @@ -163,6 +164,7 @@ markers = [ "requires_ado_pat: requires ADO_APM_PAT", "requires_ado_bearer: requires az CLI logged in + APM_TEST_ADO_BEARER=1", "requires_network_integration: requires APM_RUN_INTEGRATION_TESTS=1", + "requires_live_generic_fixture: requires APM_LIVE_GENERIC_PACKAGE to point at a live fixture", "requires_apm_binary: requires built apm binary on PATH (or APM_BINARY_PATH)", "requires_runtime_codex: requires codex runtime installed", "requires_runtime_copilot: requires GitHub Copilot CLI runtime installed", @@ -170,4 +172,3 @@ markers = [ "requires_inference: requires APM_RUN_INFERENCE_TESTS=1 + model API access", "req: binds a test to one or more OpenAPM req-XXX ids (spec-conformance suite)", ] - diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 87c4ca12b..485a9e516 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -191,6 +191,10 @@ def _is_network_integration() -> bool: return os.environ.get("APM_RUN_INTEGRATION_TESTS") == "1" +def _has_live_generic_fixture() -> bool: + return bool(os.environ.get("APM_LIVE_GENERIC_PACKAGE", "").strip()) + + def _is_inference_mode() -> bool: return os.environ.get("APM_RUN_INFERENCE_TESTS") == "1" @@ -221,6 +225,10 @@ def _has_runtime(name: str) -> bool: _is_network_integration, "APM_RUN_INTEGRATION_TESTS=1 not set", ), + "requires_live_generic_fixture": ( + _has_live_generic_fixture, + "APM_LIVE_GENERIC_PACKAGE not set (example: gitlab.com//)", + ), "requires_apm_binary": ( _has_apm_binary, "apm binary not found on PATH (set APM_BINARY_PATH or build via scripts/build-binary.sh)", diff --git a/tests/integration/test_live_generic_gitlab_install.py b/tests/integration/test_live_generic_gitlab_install.py new file mode 100644 index 000000000..e543557ef --- /dev/null +++ b/tests/integration/test_live_generic_gitlab_install.py @@ -0,0 +1,256 @@ +"""Live smoke test for installing from gitlab.com via the GitLab backend. + +This is intentionally separate from the hermetic GitLab REST-path coverage in +``test_gitlab_install_e2e.py``. The gap tracked by microsoft/apm#1229 is the +non-GitHub/non-ADO clone path: ``apm install gitlab.com//`` should +route through ``GitLabBackend`` (``kind=gitlab``), delegate to git, validate the +package, and stamp the lockfile with the concrete GitLab host and resolved +commit. + +The fixture repository is configured by ``APM_LIVE_GENERIC_PACKAGE`` and pinned +by ``APM_LIVE_GENERIC_EXPECTED_SHA``. Keep them unset in ordinary CI; the +scheduled/manual workflow step enables this smoke as soon as maintainers provide +a stable public APM-shaped GitLab repo, for example: + + APM_LIVE_GENERIC_PACKAGE=gitlab.com/microsoft-apm-fixtures/smoke-pkg + APM_LIVE_GENERIC_EXPECTED_SHA=<40-char commit sha> +""" + +from __future__ import annotations + +import os +import re +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +from apm_cli.models.apm_package import DependencyReference + +pytestmark = [ + pytest.mark.live, + pytest.mark.live_generic, + pytest.mark.requires_live_generic_fixture, + pytest.mark.requires_apm_binary, + pytest.mark.requires_network_integration, +] + +_LIVE_PACKAGE_ENV = "APM_LIVE_GENERIC_PACKAGE" +_LIVE_HOST_ENV = "APM_LIVE_GENERIC_HOST" +_LIVE_EXPECTED_SHA_ENV = "APM_LIVE_GENERIC_EXPECTED_SHA" +_DEFAULT_HOST = "gitlab.com" +INSTALL_TIMEOUT_SECONDS = 240 +_OUTPUT_TAIL_CHARS = 2000 +_FULL_SHA_RE = re.compile(r"^[0-9a-f]{40}$") +_SUBPROCESS_ENV_ALLOWLIST = { + "APM_E2E_TESTS", + "APM_RUN_INTEGRATION_TESTS", + "GIT_SSL_CAINFO", + "HTTPS_PROXY", + "HTTP_PROXY", + "LANG", + "LC_ALL", + "LC_CTYPE", + "NO_PROXY", + "PATH", + "REQUESTS_CA_BUNDLE", + "SSL_CERT_FILE", + "SYSTEMROOT", + "TEMP", + "TMP", + "TMPDIR", + "WINDIR", +} +_TOKEN_ENV_DENYLIST = { + "ACTIONS_RUNTIME_TOKEN", + "GITHUB_TOKEN", + "GITLAB_APM_PAT", + "GITLAB_TOKEN", +} + + +def _configured_package() -> DependencyReference: + raw = os.environ.get(_LIVE_PACKAGE_ENV, "").strip() + if not raw: + pytest.fail( + f"{_LIVE_PACKAGE_ENV} is not set; " + f"set {_LIVE_PACKAGE_ENV}=gitlab.com//" + ) + + dep = DependencyReference.parse(raw) + expected_host = os.environ.get(_LIVE_HOST_ENV, _DEFAULT_HOST).strip() or _DEFAULT_HOST + if dep.host != expected_host: + pytest.fail( + f"{_LIVE_PACKAGE_ENV} must point at {expected_host}; " + f"parsed host={dep.host!r} from {raw!r}" + ) + if dep.is_virtual: + pytest.fail(f"{_LIVE_PACKAGE_ENV} must be an APM-shaped repo, not a virtual path: {raw!r}") + return dep + + +def _expected_sha() -> str: + raw = os.environ.get(_LIVE_EXPECTED_SHA_ENV, "").strip().lower() + if not raw: + pytest.fail( + f"{_LIVE_EXPECTED_SHA_ENV} must be set to the fixture's pinned " + "40-character commit SHA" + ) + if not _FULL_SHA_RE.fullmatch(raw): + pytest.fail(f"{_LIVE_EXPECTED_SHA_ENV} must be a full commit SHA, got {raw!r}") + return raw + + +def _write_consumer_project(project: Path, package_ref: str) -> None: + project.mkdir(parents=True) + (project / "apm.yml").write_text( + yaml.safe_dump( + { + "name": "live-generic-gitlab-smoke", + "version": "0.1.0", + "target": "copilot", + "dependencies": {"apm": [package_ref], "mcp": []}, + }, + sort_keys=False, + ), + encoding="utf-8", + ) + + +def _env_with_isolated_home(home: Path) -> dict[str, str]: + env = { + key: value + for key, value in os.environ.items() + if key.upper() in _SUBPROCESS_ENV_ALLOWLIST and key.upper() not in _TOKEN_ENV_DENYLIST + } + env["HOME"] = str(home) + env["GIT_TERMINAL_PROMPT"] = "0" + env["GIT_CONFIG_NOSYSTEM"] = "1" + env["NO_COLOR"] = "1" + env["APM_E2E_TESTS"] = "1" + if sys.platform == "win32": + env["USERPROFILE"] = str(home) + if system_root := os.environ.get("SYSTEMROOT"): + env["SYSTEMROOT"] = system_root + return env + + +def _tail_output(text: str) -> str: + if len(text) <= _OUTPUT_TAIL_CHARS: + return text + return f"[truncated to last {_OUTPUT_TAIL_CHARS} chars]\n{text[-_OUTPUT_TAIL_CHARS:]}" + + +def _read_lockfile(project: Path) -> dict[str, object]: + lock_path = project / "apm.lock.yaml" + assert lock_path.exists(), "apm install did not create apm.lock.yaml" + data = yaml.safe_load(lock_path.read_text(encoding="utf-8")) + assert isinstance(data, dict), ( + f"apm.lock.yaml must contain a YAML mapping, got {type(data).__name__}: {data!r}" + ) + return data + + +def _locked_dep(lockfile: dict[str, object], expected: DependencyReference) -> dict | None: + deps = lockfile.get("dependencies") + assert isinstance(deps, list), ( + f"apm.lock.yaml dependencies must be a list, got {type(deps).__name__}: {deps!r}" + ) + for dep in deps: + assert isinstance(dep, dict), f"lockfile dependency entry must be a mapping: {dep!r}" + if dep.get("host") == expected.host and dep.get("repo_url") == expected.repo_url: + return dep + return None + + +def _run_install(apm_binary_path: Path, project: Path, fake_home: Path) -> subprocess.CompletedProcess: + return subprocess.run( + [str(apm_binary_path), "install"], + cwd=project, + capture_output=True, + text=True, + timeout=INSTALL_TIMEOUT_SECONDS, + env=_env_with_isolated_home(fake_home), + check=False, + ) + + +def _assert_install_succeeded(result: subprocess.CompletedProcess, package_ref: str) -> None: + assert result.returncode == 0, ( + "live GitLab install failed\n" + f"package: {package_ref}\n" + f"stdout:\n{_tail_output(result.stdout)}\n" + f"stderr:\n{_tail_output(result.stderr)}" + ) + + +def _assert_install_output_mentions_success( + result: subprocess.CompletedProcess, dep: DependencyReference +) -> None: + package_name = dep.repo_url.rsplit("/", 1)[-1].lower() + output = f"{result.stdout}\n{result.stderr}".lower() + assert "installed" in output or package_name in output, ( + "live GitLab install output did not mention an install summary or package name; " + f"stdout:\n{_tail_output(result.stdout)}\n" + f"stderr:\n{_tail_output(result.stderr)}" + ) + + +def _assert_installed_package_manifest(project: Path) -> None: + installed_manifests = list((project / "apm_modules").rglob("apm.yml")) + assert installed_manifests, "install did not materialize an APM package under apm_modules/" + for manifest in installed_manifests: + data = yaml.safe_load(manifest.read_text(encoding="utf-8")) + if isinstance(data, dict) and isinstance(data.get("name"), str) and data["name"]: + return + raise AssertionError( + f"installed apm.yml files did not contain package names: {installed_manifests}" + ) + + +def test_live_gitlab_install_clones_validates_and_stamps_lockfile( + apm_binary_path: Path, tmp_path: Path +) -> None: + """Run ``apm install`` against a real gitlab.com repo through GitLabBackend.""" + dep = _configured_package() + expected_sha = _expected_sha() + project = tmp_path / "consumer" + fake_home = tmp_path / "home" + fake_home.mkdir() + + package_ref = dep.to_canonical() + _write_consumer_project(project, package_ref) + + result = _run_install(apm_binary_path, project, fake_home) + _assert_install_succeeded(result, package_ref) + _assert_install_output_mentions_success(result, dep) + _assert_installed_package_manifest(project) + + lock_path = project / "apm.lock.yaml" + first_lock_text = lock_path.read_text(encoding="utf-8") + lockfile = _read_lockfile(project) + locked = _locked_dep(lockfile, dep) + assert locked is not None, ( + f"lockfile did not contain {dep.host}/{dep.repo_url}; " + f"dependencies={lockfile.get('dependencies')}" + ) + assert locked.get("host") == dep.host + assert locked.get("repo_url") == dep.repo_url + resolved_commit = locked.get("resolved_commit") + assert isinstance(resolved_commit, str) and resolved_commit, ( + f"resolved_commit must be a non-empty string in lockfile entry: {locked}" + ) + assert _FULL_SHA_RE.fullmatch(resolved_commit), ( + f"resolved_commit is not a full commit SHA: {resolved_commit!r}" + ) + assert resolved_commit == expected_sha, ( + f"resolved_commit did not match {_LIVE_EXPECTED_SHA_ENV}: " + f"expected {expected_sha}, got {resolved_commit}" + ) + + second_result = _run_install(apm_binary_path, project, fake_home) + _assert_install_succeeded(second_result, package_ref) + second_lock_text = lock_path.read_text(encoding="utf-8") + assert second_lock_text == first_lock_text, "second install changed apm.lock.yaml" diff --git a/tests/integration/test_marker_registry_sync.py b/tests/integration/test_marker_registry_sync.py index ab3fca581..c6bf54e61 100644 --- a/tests/integration/test_marker_registry_sync.py +++ b/tests/integration/test_marker_registry_sync.py @@ -85,13 +85,13 @@ def _conftest_marker_names() -> set[str]: def _gating_markers_in_pyproject() -> set[str]: - """Subset of pyproject markers that gate execution (requires_* + live). + """Subset of pyproject markers that gate execution. ``integration``, ``slow``, ``benchmark`` are taxonomy markers, not gates, and are intentionally excluded from the docs registry. """ names = _pyproject_marker_names() - return {n for n in names if n.startswith("requires_") or n == "live"} + return {n for n in names if n.startswith("requires_") or n in {"live", "live_generic"}} # --------------------------------------------------------------------------- @@ -191,6 +191,7 @@ def test_integration_tests_use_pytestmark_not_runtime_self_skip() -> None: """ gate_env_vars = ( "APM_E2E_TESTS", + "APM_LIVE_GENERIC_PACKAGE", "APM_RUN_INTEGRATION_TESTS", "APM_RUN_INFERENCE_TESTS", "APM_TEST_ADO_BEARER", diff --git a/tests/unit/test_live_generic_gitlab_install_helpers.py b/tests/unit/test_live_generic_gitlab_install_helpers.py new file mode 100644 index 000000000..83a3a70c7 --- /dev/null +++ b/tests/unit/test_live_generic_gitlab_install_helpers.py @@ -0,0 +1,41 @@ +"""Hermetic coverage for the live GitLab smoke-test helpers.""" + +from __future__ import annotations + +from pathlib import Path + +from tests.integration import test_live_generic_gitlab_install as live_gitlab + + +def test_env_with_isolated_home_strips_token_vars(monkeypatch, tmp_path: Path) -> None: + monkeypatch.setenv("PATH", "/usr/bin") + monkeypatch.setenv("GITLAB_APM_PAT", "glpat-secret") + monkeypatch.setenv("GITLAB_TOKEN", "glpat-fallback") + monkeypatch.setenv("GITHUB_TOKEN", "gh-actions-token") + monkeypatch.setenv("ACTIONS_RUNTIME_TOKEN", "actions-runtime-token") + monkeypatch.setenv("GIT_CONFIG_GLOBAL", "/tmp/leaky-gitconfig") + monkeypatch.setenv("APM_RUN_INTEGRATION_TESTS", "1") + + env = live_gitlab._env_with_isolated_home(tmp_path) + + assert env["HOME"] == str(tmp_path) + assert env["GIT_TERMINAL_PROMPT"] == "0" + assert env["GIT_CONFIG_NOSYSTEM"] == "1" + assert env["NO_COLOR"] == "1" + assert env["APM_E2E_TESTS"] == "1" + assert env["APM_RUN_INTEGRATION_TESTS"] == "1" + assert env["PATH"] == "/usr/bin" + assert "GITLAB_APM_PAT" not in env + assert "GITLAB_TOKEN" not in env + assert "GITHUB_TOKEN" not in env + assert "ACTIONS_RUNTIME_TOKEN" not in env + assert "GIT_CONFIG_GLOBAL" not in env + + +def test_tail_output_truncates_long_output() -> None: + text = "a" * (live_gitlab._OUTPUT_TAIL_CHARS + 10) + + result = live_gitlab._tail_output(text) + + assert result.startswith("[truncated to last") + assert result.endswith("a" * live_gitlab._OUTPUT_TAIL_CHARS)