diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ab1f603159c..6f17d6aa458 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -121,7 +121,7 @@ serverless lambda tests: trigger: project: DataDog/datadog-lambda-python strategy: depend - branch: main + branch: emmett.butler/serverless-wheels needs: - job: "upload manylinux2014" rules: diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 406d786eb3f..e19bb8257fd 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -378,10 +378,19 @@ download_dependency_wheels: - "pywheels/*.whl" # Aggregate and validate all built wheels -"ddtrace package": +.package_base: image: registry.ddbuild.io/images/mirror/python:3.14.0 tags: [ "arch:amd64" ] stage: package + script: + - pip install 'packaging>=24.0,<26' # ci-deps: allow + artifacts: + paths: + - "pywheels/*.whl" + - "pywheels/*.tar.gz" + +"ddtrace package": + extends: .package_base needs: - "build linux" - "build macos" @@ -389,9 +398,14 @@ download_dependency_wheels: - "collect windows wheels" - "package version" script: - - pip install 'packaging>=24.0,<26' # ci-deps: allow + - !reference [.package_base, script] - .gitlab/validate-ddtrace-package.py pywheels - artifacts: - paths: - - "pywheels/*.whl" - - "pywheels/*.tar.gz" + +"ddtrace package serverless": + extends: .package_base + needs: + - "build linux serverless" + - "package version" + script: + - !reference [.package_base, script] + - .gitlab/validate-ddtrace-package.py pywheels --mode=serverless diff --git a/.gitlab/release.yml b/.gitlab/release.yml index 47d5b4702d6..7f5957ab287 100644 --- a/.gitlab/release.yml +++ b/.gitlab/release.yml @@ -26,8 +26,7 @@ variables: - pip install -r ci/requirements/ci.txt - python -m twine check --strict pywheels/* script: - - rm -rf pywheels/ddtrace_serverless* - - python -m twine upload --repository ${PYPI_REPOSITORY} pywheels/* + - python -m twine upload --repository ${PYPI_REPOSITORY} pywheels/ddtrace-* artifacts: paths: - pywheels/*.whl diff --git a/.gitlab/scripts/build-wheel-helpers.sh b/.gitlab/scripts/build-wheel-helpers.sh index e4676c8c9b8..dc3bee8017f 100755 --- a/.gitlab/scripts/build-wheel-helpers.sh +++ b/.gitlab/scripts/build-wheel-helpers.sh @@ -136,6 +136,7 @@ finalize() { export TMP_WHEEL_FILE=$(ls ${TMP_WHEEL_DIR}/*.whl | head -n 1) WHEEL_BASENAME=$(basename "${TMP_WHEEL_FILE}") mv "${TMP_WHEEL_FILE}" "${FINAL_WHEEL_DIR}/" + ls -al "${FINAL_WHEEL_DIR}" export FINAL_WHEEL_FILE="${FINAL_WHEEL_DIR}/${WHEEL_BASENAME}" section_end "finalize_wheel" } diff --git a/.gitlab/validate-ddtrace-package.py b/.gitlab/validate-ddtrace-package.py index c5613abac79..6ef5c304963 100755 --- a/.gitlab/validate-ddtrace-package.py +++ b/.gitlab/validate-ddtrace-package.py @@ -16,6 +16,7 @@ PACKAGE_VERSION: Version from pyproject.toml (set by "package version" job) """ +import argparse import os from pathlib import Path import sys @@ -41,21 +42,25 @@ SERVERLESS_PLATFORMS = [p for p in BASE_PLATFORMS if "linux" in p] -def build_expected_set(version: str) -> set[tuple[str, str, str]]: +def build_expected_set(version: str, args: argparse.Namespace) -> set[tuple[str, str, str]]: """Build set of expected (version, python_tag, platform, flavor) tuples.""" expected: set[tuple[str, str, str, str]] = set() for py_tag in PYTHON_TAGS: - for platform in BASE_PLATFORMS: - expected.add((version, py_tag, platform, "")) - # Add win_arm64 for Python 3.11+ - if py_tag in WIN_ARM64_PYTHON_TAGS: - expected.add((version, py_tag, "win_arm64", "")) + if args.mode == "serverless": + for platform in SERVERLESS_PLATFORMS: + expected.add((version, py_tag, platform, "_serverless")) + else: + for platform in BASE_PLATFORMS: + expected.add((version, py_tag, platform, "")) + # Add win_arm64 for Python 3.11+ + if py_tag in WIN_ARM64_PYTHON_TAGS: + expected.add((version, py_tag, "win_arm64", "")) return expected def reconstruct_wheel_filename(version: str, python_tag: str, platform: str, flavor: str) -> str: """Reconstruct wheel filename from components.""" - package_name = f"ddtrace{flavor.replace('-', '_')}" + package_name = f"ddtrace{flavor}" return f"{package_name}-{version}-{python_tag}-{python_tag}-{platform}.whl" @@ -101,7 +106,6 @@ def parse_actual_wheels(wheels_dir: str) -> tuple[set[tuple[str, str, str, str]] for wheel_file in sorted(Path(wheels_dir).glob("*.whl")): try: name, version, build, tags = parse_wheel_filename(wheel_file.name) - flavor = name.replace("ddtrace", "").replace("_", "-") # Extract python tag - all tags should have the same interpreter py_tag = next(iter(tags)).interpreter @@ -110,12 +114,13 @@ def parse_actual_wheels(wheels_dir: str) -> tuple[set[tuple[str, str, str, str]] # We know: name=ddtrace, abi=python tag (e.g., cp310) # So platform is everything after: ddtrace-{version}-{python}-{python}- wheel_base = wheel_file.name.replace(".whl", "") - marker = f"{name}-{version}-{py_tag}-{py_tag}-" + marker = f"{name.replace('-', '_')}-{version}-{py_tag}-{py_tag}-" if marker in wheel_base: platform = wheel_base.split(marker)[1] else: - raise ValueError(f"Cannot parse platform from {wheel_file.name}") + raise ValueError(f"Cannot parse platform from {wheel_file.name} - searched for marker {marker}") + flavor = name.replace("ddtrace", "").replace("-", "_") actual.add((str(version), py_tag, platform, flavor)) except Exception as e: errors.append(f"{wheel_file.name}: {e}") @@ -135,12 +140,9 @@ def identify_version_mismatches( return mismatches -def main() -> None: - """Main validation function.""" - # Get arguments - wheels_dir = sys.argv[1] if len(sys.argv) > 1 else "pywheels" +def main(args: argparse.Namespace) -> None: + wheels_dir = args.wheels_dir - # Get version from environment package_version: str | None = os.environ.get("PACKAGE_VERSION") if not package_version: print("[ERROR] PACKAGE_VERSION not set. Ensure 'package version' job ran.") @@ -167,15 +169,17 @@ def main() -> None: print() # Phase 2: SDist Validation - print("[Phase 2] SDist Validation") - sdist_ok, sdist_msg, sdist_name = validate_sdist(wheels_dir, package_version) - if not sdist_ok: - print(f"✗ {sdist_msg}") - errors.append(f"SDist validation: {sdist_msg}") - else: - print(f"✓ Found sdist: {sdist_name}") - print(f"✓ {sdist_msg}") - print() + sdist_ok = False + if args.mode != "serverless": + print("[Phase 2] SDist Validation") + sdist_ok, sdist_msg, sdist_name = validate_sdist(wheels_dir, package_version) + if not sdist_ok: + print(f"✗ {sdist_msg}") + errors.append(f"SDist validation: {sdist_msg}") + else: + print(f"✓ Found sdist: {sdist_name}") + print(f"✓ {sdist_msg}") + print() # Phase 3: Parse Actual Wheels print("[Phase 3] Parsing Actual Wheels") @@ -194,7 +198,7 @@ def main() -> None: # Phase 4: Build Expected Set print("[Phase 4] Building Expected Set") - expected_set = build_expected_set(package_version) + expected_set = build_expected_set(package_version, args) print(f"Expected {len(expected_set)} wheels:") print(f" - {len(PYTHON_TAGS)} Python versions (cp39-cp314)") print(f" - {len(BASE_PLATFORMS)} base platforms") @@ -281,4 +285,8 @@ def main() -> None: if __name__ == "__main__": - main() + parser = argparse.ArgumentParser(prog="Validate DDTrace Package") + parser.add_argument("--mode", choices=["main", "serverless"], default="main") + parser.add_argument("wheels_dir", nargs="?", default="pywheels") + args = parser.parse_args() + main(args)