diff --git a/.github/workflows/package-production.yml b/.github/workflows/package-production.yml index 0abb8e4a..6aa8d6c8 100644 --- a/.github/workflows/package-production.yml +++ b/.github/workflows/package-production.yml @@ -11,8 +11,30 @@ permissions: contents: read jobs: + build-rustls-windows: + name: Build Rustls package (win-x64) + runs-on: windows-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build and test Rustls win-x64 package + shell: pwsh + run: ./scripts/package/Build-RustlsTls13ImplementationPackage.ps1 -RuntimeIdentifier win-x64 + + - name: Upload Rustls win-x64 package + uses: actions/upload-artifact@v4 + with: + name: rustls-tls13-win-x64 + if-no-files-found: error + path: | + artifacts/packages/*rustls-tls13*.win-x64.plabpkg + artifacts/packages/*rustls-tls13*.win-x64.plabpkg.build-attestation.json + build-packages: name: Build package artifacts + needs: build-rustls-windows runs-on: ubuntu-latest steps: @@ -32,9 +54,15 @@ jobs: implementations/quic-go-http3/go.sum executors/quic-go-raw-load/source/go.sum + - name: Download Rustls win-x64 package + uses: actions/download-artifact@v4 + with: + name: rustls-tls13-win-x64 + path: artifacts/packages + - name: Build all ProtocolLab component packages shell: pwsh - run: ./scripts/package/Build-AllProtocolLabComponentPackages.ps1 -Clean + run: ./scripts/package/Build-AllProtocolLabComponentPackages.ps1 -ImportedPackageBuildKey 'implementations/rustls-tls13|win-x64' - name: Upload package production artifacts uses: actions/upload-artifact@v4 diff --git a/scripts/package/Build-AllProtocolLabComponentPackages.ps1 b/scripts/package/Build-AllProtocolLabComponentPackages.ps1 index 2a55aa40..09e60a7d 100644 --- a/scripts/package/Build-AllProtocolLabComponentPackages.ps1 +++ b/scripts/package/Build-AllProtocolLabComponentPackages.ps1 @@ -4,7 +4,9 @@ param( [string]$OutputRoot = (Join-Path $Root 'artifacts/packages'), - [switch]$Clean + [switch]$Clean, + + [string[]]$ImportedPackageBuildKey = @() ) $ErrorActionPreference = 'Stop' @@ -69,6 +71,17 @@ function ConvertTo-StringArray { return @($Value | ForEach-Object { [string]$_ }) } +function Get-PackageBuildKey { + param([Parameter(Mandatory)]$Build) + + $arguments = ConvertTo-StringArray -Value $Build.arguments + if ($arguments.Count -eq 0) { + return [string]$Build.componentPath + } + + return '{0}|{1}' -f $Build.componentPath, ($arguments -join ',') +} + function Get-ProvidedIds { param([Parameter(Mandatory)]$Manifest) @@ -257,6 +270,10 @@ $Root = (Resolve-Path $Root).Path $OutputRoot = [System.IO.Path]::GetFullPath($OutputRoot) Assert-PathIsUnderRoot -CandidatePath $OutputRoot -ExpectedRoot $Root +if ($Clean -and $ImportedPackageBuildKey.Count -gt 0) { + throw 'Clean cannot be combined with ImportedPackageBuildKey because cleaning would delete the transferred package artifacts.' +} + New-Item -ItemType Directory -Force -Path $OutputRoot | Out-Null if ($Clean) { @@ -417,10 +434,101 @@ $packageBuilds = @( [pscustomobject]@{ componentPath = 'scenarios/masque-connect-udp-performance'; script = 'Build-MasqueConnectUdpScenarioPackage.ps1'; arguments = @() } ) +$packageBuildsByKey = @{} +foreach ($build in $packageBuilds) { + $buildKey = Get-PackageBuildKey -Build $build + if ($packageBuildsByKey.ContainsKey($buildKey)) { + throw "Duplicate package build key '$buildKey'." + } + + $packageBuildsByKey[$buildKey] = $build +} + +$importedBuildKeys = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) +foreach ($buildKey in $ImportedPackageBuildKey) { + if (-not $packageBuildsByKey.ContainsKey($buildKey)) { + throw "Imported package build key was not found: '$buildKey'." + } + + [void]$importedBuildKeys.Add($buildKey) +} + $builderResults = [System.Collections.Generic.List[object]]::new() $builtArtifacts = [System.Collections.Generic.List[System.IO.FileInfo]]::new() +$importedArtifactPaths = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) + +if ($importedBuildKeys.Count -gt 0) { + $currentCommit = (& git -C $Root rev-parse HEAD).Trim() + $matchedImportedBuildKeys = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) + + foreach ($artifact in @(Get-ChildItem -LiteralPath $OutputRoot -File -Filter '*.plabpkg' | Sort-Object Name)) { + $attestationPath = "$($artifact.FullName).build-attestation.json" + if (-not (Test-Path -LiteralPath $attestationPath -PathType Leaf)) { + throw "$($artifact.Name): imported package is missing build attestation '$attestationPath'." + } + + $attestation = Get-Content -LiteralPath $attestationPath -Raw | ConvertFrom-Json + if (-not [string]::Equals([string]$attestation.source.commitSha, $currentCommit, [System.StringComparison]::OrdinalIgnoreCase)) { + throw "$($artifact.Name): imported package commit '$($attestation.source.commitSha)' does not match aggregation commit '$currentCommit'." + } + + $actualHash = (Get-FileHash -LiteralPath $artifact.FullName -Algorithm SHA256).Hash.ToLowerInvariant() + if (-not [string]::Equals([string]$attestation.package.sha256, $actualHash, [System.StringComparison]::OrdinalIgnoreCase)) { + throw "$($artifact.Name): imported package hash does not match its build attestation." + } + + $candidateBuilds = @( + $packageBuilds | Where-Object { + [string]::Equals([string]$_.componentPath, [string]$attestation.source.componentPath, [System.StringComparison]::OrdinalIgnoreCase) + } + ) + if ($candidateBuilds.Count -gt 1) { + $candidateBuilds = @( + $candidateBuilds | Where-Object { + (ConvertTo-StringArray -Value $_.arguments) -contains [string]$attestation.build.runtimeIdentifier + } + ) + } + + if ($candidateBuilds.Count -ne 1) { + throw "$($artifact.Name): imported package could not be associated with exactly one package build." + } + + $matchingBuild = $candidateBuilds[0] + $matchingBuildKey = Get-PackageBuildKey -Build $matchingBuild + if (-not $importedBuildKeys.Contains($matchingBuildKey)) { + throw "$($artifact.Name): imported artifact maps to unrequested build '$matchingBuildKey'." + } + + $attestation.package.materializationPath = [System.IO.Path]::GetFullPath($artifact.FullName) + $attestation.package.buildAttestationPath = [System.IO.Path]::GetFullPath($attestationPath) + $attestation | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $attestationPath -Encoding utf8NoBOM + + [void]$builtArtifacts.Add($artifact) + [void]$importedArtifactPaths.Add($artifact.FullName) + [void]$matchedImportedBuildKeys.Add($matchingBuildKey) + [void]$builderResults.Add([pscustomobject]@{ + componentPath = $matchingBuild.componentPath + script = $matchingBuild.script + arguments = ConvertTo-StringArray -Value $matchingBuild.arguments + artifacts = @($artifact.Name) + status = 'passed' + }) + } + + foreach ($buildKey in $importedBuildKeys) { + if (-not $matchedImportedBuildKeys.Contains($buildKey)) { + throw "No transferred package artifact matched imported build '$buildKey'." + } + } +} foreach ($build in $packageBuilds) { + $buildKey = Get-PackageBuildKey -Build $build + if ($importedBuildKeys.Contains($buildKey)) { + continue + } + $scriptPath = Join-Path $PSScriptRoot $build.script if (-not (Test-Path -LiteralPath $scriptPath -PathType Leaf)) { throw "Package build script not found: $scriptPath" @@ -440,7 +548,7 @@ foreach ($build in $packageBuilds) { $artifacts = @( Get-ChildItem -LiteralPath $OutputRoot -File -Filter '*.plabpkg' | - Where-Object { $_.LastWriteTimeUtc -ge $startTime } | + Where-Object { $_.LastWriteTimeUtc -ge $startTime -and -not $importedArtifactPaths.Contains($_.FullName) } | Sort-Object FullName ) diff --git a/scripts/package/Build-RustlsTls13ImplementationPackage.ps1 b/scripts/package/Build-RustlsTls13ImplementationPackage.ps1 index 8415301b..f0bbb1f3 100644 --- a/scripts/package/Build-RustlsTls13ImplementationPackage.ps1 +++ b/scripts/package/Build-RustlsTls13ImplementationPackage.ps1 @@ -3,32 +3,25 @@ param( [ValidateSet('win-x64','linux-x64')][string]$RuntimeIdentifier='win-x64', [string]$Root=(Resolve-Path (Join-Path $PSScriptRoot '../..')).Path, [string]$OutputRoot=(Join-Path $Root 'artifacts/packages'), - [string]$Toolchain='stable-x86_64-pc-windows-gnu', + [string]$Toolchain='', [switch]$AllowDirtySource ) $ErrorActionPreference='Stop' $Root=[IO.Path]::GetFullPath($Root);$OutputRoot=[IO.Path]::GetFullPath($OutputRoot) $componentName='rustls-tls13';$componentRoot=Join-Path $Root "implementations/$componentName";$sourceRoot=Join-Path $componentRoot 'source' +if($RuntimeIdentifier-eq'win-x64'-and-not$IsWindows){throw 'The win-x64 rustls package must be built and tested on Windows.'} +if($RuntimeIdentifier-eq'linux-x64'-and-not$IsLinux){throw 'The linux-x64 rustls package must be built and tested on Linux.'} +if([string]::IsNullOrWhiteSpace($Toolchain)){$Toolchain=if($IsWindows){'stable-x86_64-pc-windows-gnu'}else{'stable'}} & cargo "+$Toolchain" test --locked --manifest-path (Join-Path $sourceRoot 'Cargo.toml') if($LASTEXITCODE-ne 0){throw 'rustls TLS 1.3 target tests failed.'} $rid=switch($RuntimeIdentifier){ 'win-x64'{@{os='windows';arch='x64';target=$null;name='rustls-tls13.exe';source='target/release/protocol-lab-rustls-tls13-target.exe'}} - 'linux-x64'{@{os='linux';arch='x64';target='x86_64-unknown-linux-musl';name='rustls-tls13';source='target/x86_64-unknown-linux-musl/release/protocol-lab-rustls-tls13-target'}} + 'linux-x64'{@{os='linux';arch='x64';target=$null;name='rustls-tls13';source='target/release/protocol-lab-rustls-tls13-target'}} } $buildArgs=@("+$Toolchain",'build','--locked','--release','--manifest-path',(Join-Path $sourceRoot 'Cargo.toml')) -if($rid.target){$buildArgs+=@('--target',$rid.target)} -$savedLinker=$env:CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER -try{ - if($RuntimeIdentifier-eq'linux-x64'-and$IsWindows){ - $sysroot=& rustc "+$Toolchain" --print sysroot - if($LASTEXITCODE-ne 0){throw 'Unable to resolve the pinned Rust sysroot.'} - $env:CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER=Join-Path $sysroot 'lib/rustlib/x86_64-pc-windows-gnu/bin/rust-lld.exe' - if(-not(Test-Path $env:CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER)){throw 'rust-lld is unavailable for the Linux musl package build.'} - } - & cargo @buildArgs - if($LASTEXITCODE-ne 0){throw "rustls TLS 1.3 target build failed for $RuntimeIdentifier."} -}finally{$env:CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER=$savedLinker} +& cargo @buildArgs +if($LASTEXITCODE-ne 0){throw "rustls TLS 1.3 target build failed for $RuntimeIdentifier."} $staging=Join-Path $OutputRoot "$componentName/$RuntimeIdentifier";$packageRoot=Join-Path $staging 'package' Remove-Item -LiteralPath $staging -Recurse -Force -ErrorAction SilentlyContinue New-Item -ItemType Directory -Force (Join-Path $packageRoot "bin/$RuntimeIdentifier"),(Join-Path $packageRoot 'implementations'),(Join-Path $packageRoot 'certs')|Out-Null diff --git a/scripts/package/README.md b/scripts/package/README.md index c6083c99..1189a64e 100644 --- a/scripts/package/README.md +++ b/scripts/package/README.md @@ -36,6 +36,13 @@ Package production fails unless every package has exactly one valid, parity-eligible clean-source build attestation. The package index and validation summary record each attestation artifact, its SHA-256, and source commit. +The package-production workflow builds and tests the Rustls `win-x64` package +on Windows, transfers that package and attestation to the Ubuntu aggregation +job, and builds the Rustls `linux-x64` package natively on Linux. The aggregator +uses `-ImportedPackageBuildKey` to verify the transferred artifact belongs to +the same commit, rebase its local attestation paths, and include it in the same +index, checksum manifest, and validation summary as every other package. + Use the component-specific wrappers when iterating on one package: ```powershell