From 80e34215f2338d9acbd6b6427a8d0cb942614f6a Mon Sep 17 00:00:00 2001 From: Willy Zhang Date: Tue, 28 Apr 2026 15:32:24 -0700 Subject: [PATCH] ci: Implement reproducible Nix environment for integration tests This commit introduces a parallel CI pipeline using a Nix-based reproducible development environment. It achieves dev-prod parity by ensuring local developers and the CI runner use the exact same toolchains and dependencies. Key changes: - `flake.nix`: Defines a reproducible `devShell` containing essential build tools (Bazel, Go, Python, OpenSSL) and natively configures the `libp11` PKCS11 engine for SoftHSM integration. - `.github/workflows/main.yml`: Adds a parallel job matrix to run integration tests on the new `ot-provisioning-nix-runner`. It injects environment variables via `env` to bypass Nix's environment stripping, and skips physical FPGA hardware tests (`FPGA=skip`). - `setup.sh`: Improves host backwards-compatibility by dynamically falling back to `libncursesw6` and `libtinfo6` on newer Ubuntu distributions (like 24.04). - `nix/ci-machine.nix` & `nix/README.md`: Adds the declarative NixOS infrastructure configuration for the future provisioning of the CI runner itself. --- .github/workflows/main.yml | 63 ++++++++++++++++++++++++++++++--- .gitignore | 4 +++ flake.lock | 27 ++++++++++++++ flake.nix | 46 ++++++++++++++++++++++++ nix/README.md | 36 +++++++++++++++++++ nix/ci-machine.nix | 72 ++++++++++++++++++++++++++++++++++++++ setup.sh | 14 ++++++-- 7 files changed, 255 insertions(+), 7 deletions(-) create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 nix/README.md create mode 100644 nix/ci-machine.nix diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d5b41710..4a2e7642 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,7 +18,7 @@ jobs: - name: Run linter checks run: bazelisk test //quality/... --test_output=errors - # Build and test everything in the codebase (except the FPGA tests). + # EXISTING MACHINE: Build and test everything (Original Flow) build_and_test: runs-on: ot-provisioning-self-hosted timeout-minutes: 80 @@ -43,7 +43,44 @@ jobs: # Skip running lint tests as those are run in a previous stage. run: bazelisk test --test_tag_filters=-lint,-fpga //... - # Run integration tests. + # NEW NIX MACHINE: Build and test everything using Nix + build_and_test_nix: + runs-on: ot-provisioning-nix-runner + timeout-minutes: 80 + steps: + - uses: actions/checkout@v4 + with: + lfs: true + - name: Build everything (Nix) + run: | + nix develop --command bazelisk build \ + --action_env=LD_LIBRARY_PATH \ + --host_action_env=LD_LIBRARY_PATH \ + //... + nix develop --command bazelisk build \ + --action_env=LD_LIBRARY_PATH \ + --host_action_env=LD_LIBRARY_PATH \ + --//src/pa/services:use_vendor_shim //src/pa/services:pa + nix develop --command bazelisk run \ + --action_env=LD_LIBRARY_PATH \ + --host_action_env=LD_LIBRARY_PATH \ + //release:release -- \ + --norelease \ + --copy "${PWD}/artifacts-nix" \ + "no_tag_not_a_release" + - name: Upload build artifacts (Nix) + uses: actions/upload-artifact@v4 + with: + name: artifacts-nix + path: artifacts-nix/* + - name: Test everything (Nix) + run: | + nix develop --command bazelisk test \ + --action_env=LD_LIBRARY_PATH \ + --host_action_env=LD_LIBRARY_PATH \ + --test_tag_filters=-lint,-fpga //... + + # EXISTING MACHINE: Run integration tests (Original Flow) integration_tests: runs-on: ot-provisioning-self-hosted timeout-minutes: 60 @@ -54,8 +91,6 @@ jobs: - uses: actions/checkout@v4 with: lfs: true - # TODO(timothytrippel): It is unclear why the `lfs: true` configuration - # above is not sufficient on self hosted runners. - name: Initialize LFS objects run: git lfs pull - name: Run provisioning appliance load test @@ -68,3 +103,23 @@ jobs: run: OT_PROV_ORCHESTRATOR_PATH="${OT_PROV_ORCHESTRATOR_PATH}" OT_PROV_ORCHESTRATOR_UNPACK="${OT_PROV_ORCHESTRATOR_UNPACK}" OPENTITAN_VAR_DIR=$(pwd)/.otvar-dev ./tests/run_ate_test.sh - name: Run integration tests (Thales HSM) run: OT_PROV_ORCHESTRATOR_PATH="${OT_PROV_ORCHESTRATOR_PATH}" OT_PROV_ORCHESTRATOR_UNPACK="${OT_PROV_ORCHESTRATOR_UNPACK}" OPENTITAN_VAR_DIR=$(pwd)/.otvar-prod ./tests/run_ate_test.sh --prod + + # NEW NIX MACHINE: Run integration tests using Nix + integration_tests_nix: + runs-on: ot-provisioning-nix-runner + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + with: + lfs: true + - name: Initialize LFS objects + run: git lfs pull + - name: Run provisioning appliance load test (Nix) + run: nix develop --command env OPENTITAN_VAR_DIR=$(pwd)/.otvar-dev ./tests/run_pa_loadtest.sh + - name: Run provisioning appliance load test (PQ) (Nix) + run: nix develop --command env OPENTITAN_VAR_DIR=$(pwd)/.otvar-dev-pq ./tests/run_pa_loadtest.sh --pq + - name: Run TLS test (Nix) + run: nix develop --command env OPENTITAN_VAR_DIR=$(pwd)/.otvar-dev ./tests/run_tls_test.sh + - name: Run integration tests (SoftHSM2) (Nix) + run: nix develop --command env FPGA=skip OPENTITAN_VAR_DIR=$(pwd)/.otvar-dev ./tests/run_ate_test.sh + diff --git a/.gitignore b/.gitignore index 9143a4f5..b79febd6 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,7 @@ .otprod bazel-* *.swp + +# Nix configuration artifacts +.bazelrc-nix +.nix-libs/ diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000..fe08f566 --- /dev/null +++ b/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1751274312, + "narHash": "sha256-/bVBlRpECLVzjV19t5KMdMFWSwKLtb5RyXdjz3LJT+g=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "50ab793786d9de88ee30ec4e4c24fb4236fc2674", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-24.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000..88bfe224 --- /dev/null +++ b/flake.nix @@ -0,0 +1,46 @@ +{ + description = "NixOS configuration and development environment for OpenTitan Provisioning"; +inputs = { + # Pinning to a stable version for consistency + nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11"; +}; + +outputs = { self, nixpkgs, ... }: + let + system = "x86_64-linux"; + pkgs = import nixpkgs { + inherit system; + config.allowUnfree = true; + }; + + sharedTools = with pkgs; [ + # Base tools + coreutils git git-lfs curl gnutar gzip unzip + # Build tools + bazelisk go python3 python3Packages.pip cmake gnumake gcc pkg-config + # Libraries + libusb1 libftdi1 openssl libp11 ncurses5 udev stdenv.cc.cc.lib + ]; + + in + { + # --- NIXOS CONFIGURATIONS (The Hosts) --- + nixosConfigurations.ci-machine = nixpkgs.lib.nixosSystem { + inherit system; + modules = [ ./nix/ci-machine.nix ]; + }; + + # --- DEV/CI SHELL (The Builder) --- + devShells.${system}.default = pkgs.mkShell { + buildInputs = sharedTools; + shellHook = '' + echo "OpenTitan Provisioning Development Environment" + export OT_PROV_SHELL=1 + export OPENSSL_ENGINES="${pkgs.libp11}/lib/engines" + export LD_LIBRARY_PATH="${pkgs.lib.makeLibraryPath sharedTools}:$LD_LIBRARY_PATH" + export NIX_LD_LIBRARY_PATH="${pkgs.lib.makeLibraryPath sharedTools}" + export NIX_LD="${pkgs.stdenv.cc.libc}/lib/ld-linux-x86-64.so.2" + ''; + }; + }; +} diff --git a/nix/README.md b/nix/README.md new file mode 100644 index 00000000..28f42142 --- /dev/null +++ b/nix/README.md @@ -0,0 +1,36 @@ +# NixOS CI Machine Configuration + +This directory contains the NixOS configuration for the GitHub Actions runner. + +## How it Works + +This Flake provides two things: +1. **A System Configuration (`ci-machine`)**: For bare-metal deployment on the CI machine. +2. **A Development Shell (`default`)**: For developers and CI workflows to get an identical environment without installing tools globally. + +## Usage + +### For Developers (Local) +To enter the development environment with all tools (Bazel, Go, etc.) ready: +```bash +nix develop +``` + +### For CI Workflows +To run a command (like a build) using the pinned tools: +```bash +nix develop --command bazelisk build //... +``` + +### For Bare-Metal Deployment +1. **Prepare the Target Machine**: Log into `ssh ci@172.16.0.230`. +2. **Configure Token**: + ```bash + sudo mkdir -p /var/lib/secrets + echo "" | sudo tee /var/lib/secrets/github-runner-token + sudo chmod 600 /var/lib/secrets/github-runner-token + ``` +3. **Deploy**: + ```bash + sudo nixos-rebuild switch --flake .#ci-machine + ``` diff --git a/nix/ci-machine.nix b/nix/ci-machine.nix new file mode 100644 index 00000000..5e583944 --- /dev/null +++ b/nix/ci-machine.nix @@ -0,0 +1,72 @@ +{ config, pkgs, ... }: + +{ + # CI User setup + users.users.ci = { + isNormalUser = true; + extraGroups = [ "wheel" "podman" ]; + shell = pkgs.bash; + }; + + # System packages (global tools) + environment.systemPackages = with pkgs; [ + vim + git + git-lfs + curl + pciutils + usbutils + ]; + + # Podman configuration + virtualisation.podman = { + enable = true; + dockerCompat = true; + }; + + # Environment initialization + systemd.tmpfiles.rules = [ + "d /var/lib/opentitan 0755 ci users -" + ]; + + # GitHub Actions Runner Service + services.github-runners.ot-provisioning-runner = { + enable = true; + url = "https://github.com/lowRISC/opentitan-provisioning"; + tokenFile = "/var/lib/secrets/github-runner-token"; + name = "ot-provisioning-nix-runner"; # UNIQUE NAME + labels = [ "ot-provisioning-nix-runner" ]; # UNIQUE LABEL + user = "ci"; + # Essential packages for the runner to function (checkout, etc.) + extraPackages = with pkgs; [ + git + git-lfs + nix + coreutils + bash + ]; + }; + + # Allow the CI user to run sudo without a password + security.sudo.extraRules = [ + { + users = [ "ci" ]; + commands = [ { command = "ALL"; options = [ "NOPASSWD" ]; } ]; + } + ]; + + # Basic system settings + networking.hostName = "ci-machine-nix"; + time.timeZone = "UTC"; + i18n.defaultLocale = "en_US.UTF-8"; + + # Enable SSH + services.openssh.enable = true; + + # Nix settings + nix.settings.experimental-features = [ "nix-command" "flakes" ]; + + # Minimal boot and file system configuration for evaluation/verification + boot.loader.grub.enable = false; + fileSystems."/" = { device = "/dev/sda1"; fsType = "ext4"; }; +} diff --git a/setup.sh b/setup.sh index d505a1a0..f63df8a7 100755 --- a/setup.sh +++ b/setup.sh @@ -9,9 +9,17 @@ readonly REPO_TOP="$(dirname "$0")" readonly OPENTITAN_VAR_DIR="/var/lib/opentitan" sudo apt update -sed -e '/^$/d' -e '/^#/d' -e 's/#.*//' \ - < "$REPO_TOP/apt-requirements.txt" \ - | sudo xargs apt install -y +PACKAGES=$(sed -e '/^$/d' -e '/^#/d' -e 's/#.*//' < "$REPO_TOP/apt-requirements.txt") + +# Fallback to newer ncurses/tinfo versions if older ones are missing (e.g. Ubuntu 24.04+) +if ! apt-cache pkgnames "^libncursesw5$" | grep -q "libncursesw5"; then + PACKAGES=$(echo "$PACKAGES" | sed 's/libncursesw5/libncursesw6/g') +fi +if ! apt-cache pkgnames "^libtinfo5$" | grep -q "libtinfo5"; then + PACKAGES=$(echo "$PACKAGES" | sed 's/libtinfo5/libtinfo6/g') +fi + +echo "$PACKAGES" | sudo xargs apt install -y sudo apt clean go install github.com/bazelbuild/bazelisk@v1.27.0