diff --git a/.azure-pipelines/ci-build.yml b/.azure-pipelines/ci-build.yml index 444401e199..5a2b76b57d 100644 --- a/.azure-pipelines/ci-build.yml +++ b/.azure-pipelines/ci-build.yml @@ -36,6 +36,11 @@ variables: IMAGE_NAME: "public/openapi/kiota" PREVIEW_BRANCH: "refs/heads/main" TAG: "$(Build.BuildId)" + # Base URL for the authenticated Azure Artifacts feed (GraphDeveloperExperiences_Public) used to route + # package operations (version lookups + NuGet restore) through configured upstream sources instead of + # public registries. Defined once here and referenced as $(privateFeedBaseUrl) throughout the pipeline. + # NOTE: confirm this against the feed's "Connect to Feed" dialog if the feed/org/project changes. + privateFeedBaseUrl: "https://microsoftgraph.pkgs.visualstudio.com/0985d294-5762-4bc2-a565-161ef349ca3e/_packaging/GraphDeveloperExperiences_Public" parameters: - name: previewBranch @@ -94,6 +99,16 @@ resources: extends: template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates parameters: + settings: + # Network isolation policy. "CFSClean" enforces Central Feed Service isolation: package restores + # are constrained to approved/authenticated feeds and direct calls to public package registries + # (nuget.org, npmjs, pypi.org, Maven Central, etc.) are blocked. "GitHub" is a shared allow policy + # that permits github.com / api.github.com (needed for the Go dependency release lookups). The + # "Permissive" base policy is intentionally omitted so other banned-endpoint calls still fail the + # run, surfacing exactly which steps egress to public registries. + # See https://aka.ms/1espt-networkisolation + # explicitly unset until we can get an approval for our required use-cases as these policies will break the build. + # networkIsolationPolicy: Preferred,GitHub,CFSClean sdl: sourceAnalysisPool: name: Azure-Pipelines-1ESPT-ExDShared @@ -118,8 +133,27 @@ extends: clean: true submodules: true - - pwsh: $(Build.SourcesDirectory)/scripts/update-versions.ps1 + # The it/ integration-test fixtures are not used by this pipeline and their per-language + # manifests trigger Component Governance to reach public registries under network isolation. + # Remove them right after checkout so no later scan/build sees them. + - script: git -C "$(Build.SourcesDirectory)" rm -r --quiet --ignore-unmatch -- it + displayName: "Remove unused integration-test fixtures (it/)" + + # Route package-version lookups through the authenticated Azure Artifacts feed + # (GraphDeveloperExperiences_Public) so CI does not call public registries directly. + # NuGet, npm, PyPI and Maven Central are configured as upstream sources on that feed. + # PHP (Packagist), Dart (pub.dev), Go (api.github.com) and Ruby (rubygems.org) have no + # Azure Artifacts upstream type, so those lookups remain on public (allow-listed) registries. + - pwsh: > + $(Build.SourcesDirectory)/scripts/update-versions.ps1 + -NuGetServiceIndexUrl "$(privateFeedBaseUrl)/nuget/v3/index.json" + -NpmRegistryUrl "$(privateFeedBaseUrl)/npm/registry" + -PyPiSimpleIndexUrl "$(privateFeedBaseUrl)/pypi/simple" + -MavenRepositoryUrl "$(privateFeedBaseUrl)/maven/v1" displayName: "Update dependencies versions" + name: update_versions + env: + FEED_ACCESS_TOKEN: $(System.AccessToken) - pwsh: | New-Item -Path $(Build.ArtifactStagingDirectory)/AppSettings -ItemType Directory -Force -Verbose @@ -191,6 +225,9 @@ extends: clean: true submodules: true + - script: git -C "$(Build.SourcesDirectory)" rm -r --quiet --ignore-unmatch -- it + displayName: "Remove unused integration-test fixtures (it/)" + - pwsh: | Copy-Item $(Build.ArtifactStagingDirectory)/AppSettings/appsettings.json $(Build.SourcesDirectory)/src/kiota/appsettings.json -Force -Verbose displayName: Copy the appsettings.json @@ -248,7 +285,7 @@ extends: - + "@ | Set-Content -Path "$(Build.SourcesDirectory)/nuget.config" -Encoding UTF8 @@ -277,9 +314,10 @@ extends: projects: '$(Build.SourcesDirectory)\kiota.slnx' arguments: '--configuration $(BuildConfiguration) --no-build --collect:"XPlat Code Coverage"' - - pwsh: dotnet tool install --global dotnet-reportgenerator-globaltool + - pwsh: dotnet tool install --global dotnet-reportgenerator-globaltool --configfile "$(Build.SourcesDirectory)/nuget.config" condition: succeededOrFailed() displayName: "Install ReportGenerator" + name: install_reportgenerator - pwsh: reportgenerator -reports:$(Agent.TempDirectory)/**/coverage.cobertura.xml -targetdir:$(Build.SourcesDirectory)/reports/coverage -reporttypes:"HtmlInline_AzurePipelines;Cobertura" condition: succeeded() @@ -440,6 +478,8 @@ extends: - checkout: self clean: true submodules: true + - script: git -C "$(Build.SourcesDirectory)" rm -r --quiet --ignore-unmatch -- it + displayName: "Remove unused integration-test fixtures (it/)" - task: UseDotNet@2 displayName: "Use .NET 6" # needed for ESRP signing inputs: @@ -455,7 +495,23 @@ extends: inputs: version: 9.x - - script: dotnet tool install --global PowerShell + # On arm64 (Windows) PowerShell Core is not preinstalled, so it is acquired via + # `dotnet tool install`. That restore must flow through the authenticated central feed, + # so both NuGetAuthenticate and a nuget.config have to exist *before* this step. The + # config is authored with cmd rather than pwsh because pwsh is not available yet on + # this host (that is exactly what we are bootstrapping). + - task: NuGetAuthenticate@1 + displayName: "Authenticate to Azure Artifacts" + - script: | + set "CFG=$(Build.SourcesDirectory)\nuget.config" + > "%CFG%" echo ^ + >>"%CFG%" echo ^ + >>"%CFG%" echo ^ + >>"%CFG%" echo ^ + >>"%CFG%" echo ^ + >>"%CFG%" echo ^ + >>"%CFG%" echo ^ + dotnet tool install --global PowerShell --configfile "%CFG%" displayName: "Install PowerShell" condition: and(succeeded(), eq('${{ distribution.hostArchitecture }}', 'arm64')) # powershell is not installed by default on arm64 images @@ -470,6 +526,18 @@ extends: - pwsh: $(Build.SourcesDirectory)/scripts/update-version-suffix-for-source-generator.ps1 -versionSuffix "$(versionSuffix)" displayName: "Set version suffix in csproj for generators" + - pwsh: | + @" + + + + + + + + "@ | Set-Content -Path "$(Build.SourcesDirectory)/nuget.config" -Encoding UTF8 + displayName: 'Create nuget.config (central feed)' + - pwsh: dotnet publish src/kiota/kiota.csproj -c Release --runtime ${{ distribution.architecture }} -p:PublishSingleFile=true --self-contained --output $(Build.ArtifactStagingDirectory)/binaries/${{ distribution.architecture }} --version-suffix "$(versionSuffix)" -f net10.0 condition: eq(variables['isPrerelease'], 'true') displayName: publish kiota as executable @@ -655,6 +723,8 @@ extends: - checkout: self clean: true submodules: true + - script: git -C "$(Build.SourcesDirectory)" rm -r --quiet --ignore-unmatch -- it + displayName: "Remove unused integration-test fixtures (it/)" - task: UseNode@1 inputs: version: "22.x" @@ -774,6 +844,8 @@ extends: - checkout: self clean: true submodules: true + - script: git -C "$(Build.SourcesDirectory)" rm -r --quiet --ignore-unmatch -- it + displayName: "Remove unused integration-test fixtures (it/)" - task: UseNode@1 inputs: version: "22.x" @@ -1090,6 +1162,9 @@ extends: steps: - checkout: self + - script: git -C "$(Build.SourcesDirectory)" rm -r --quiet --ignore-unmatch -- it + displayName: "Remove unused integration-test fixtures (it/)" + - task: AzureCLI@2 displayName: "Login to Azure Container Registry" inputs: @@ -1124,9 +1199,17 @@ extends: displayName: "Get Kiota version number" name: getversion - - pwsh: | + # Route package-version lookups through the authenticated Azure Artifacts feed + # (GraphDeveloperExperiences_Public) so CI does not call public registries directly. + - pwsh: > ./scripts/update-versions.ps1 + -NuGetServiceIndexUrl "$(privateFeedBaseUrl)/nuget/v3/index.json" + -NpmRegistryUrl "$(privateFeedBaseUrl)/npm/registry" + -PyPiSimpleIndexUrl "$(privateFeedBaseUrl)/pypi/simple" + -MavenRepositoryUrl "$(privateFeedBaseUrl)/maven/v1" displayName: "Update dependencies versions" + env: + FEED_ACCESS_TOKEN: $(System.AccessToken) - script: | docker run --privileged --rm msgraphprodregistry.azurecr.io/tonistiigi/binfmt --install all diff --git a/scripts/update-versions.ps1 b/scripts/update-versions.ps1 index 7e675cafd2..16acd9e35c 100644 --- a/scripts/update-versions.ps1 +++ b/scripts/update-versions.ps1 @@ -1,11 +1,89 @@ +[CmdletBinding()] +param( + # NuGet v3 service index. Point at a private Azure Artifacts feed's nuget/v3/index.json to route through upstreams. + [string]$NuGetServiceIndexUrl = "https://api.nuget.org/v3/index.json", + # npm registry base URL. Point at a private feed's npm/registry endpoint to route through upstreams. + [string]$NpmRegistryUrl = "https://registry.npmjs.org", + # PyPI PEP 503/691 simple index base URL. Point at a private feed's pypi/simple endpoint to route through upstreams. + [string]$PyPiSimpleIndexUrl = "https://pypi.org/simple", + # Maven repository base URL. Point at a private feed's maven/v1 endpoint to route through upstreams. + [string]$MavenRepositoryUrl = "https://repo1.maven.org/maven2", + # Access token used to authenticate to the private feed (e.g. the pipeline System.AccessToken). + # Leave empty for anonymous access to the public registries. + [string]$FeedAccessToken = $env:FEED_ACCESS_TOKEN +) + +# Normalize trailing slashes so URL composition is predictable. +$NpmRegistryUrl = $NpmRegistryUrl.TrimEnd('/') +$PyPiSimpleIndexUrl = $PyPiSimpleIndexUrl.TrimEnd('/') +$MavenRepositoryUrl = $MavenRepositoryUrl.TrimEnd('/') + +# Azure Artifacts protocol endpoints accept the pipeline OAuth token as a Bearer token. That token must +# only ever be sent to a private Azure Artifacts feed - never to a public registry (nuget.org, npmjs.org, +# pypi.org, Maven Central, ...) where it could be leaked. Decide per-feed whether to attach it. +$script:HasFeedAccessToken = -not [string]::IsNullOrWhiteSpace($FeedAccessToken) + +# True when the URL points at an Azure Artifacts feed. Those are served from pkgs.dev.azure.com and the +# legacy *.pkgs.visualstudio.com hosts; anything else is treated as a public registry (no token sent). +function Test-IsAzureArtifactsUrl { + param([string]$Url) + if ([string]::IsNullOrWhiteSpace($Url)) { return $false } + try { + $feedHost = ([uri]$Url).Host + } + catch { + return $false + } + return $feedHost -eq "pkgs.dev.azure.com" -or $feedHost.EndsWith(".pkgs.visualstudio.com") +} + +# Build the auth headers for a feed URL: attach the Bearer token only when a token is available AND the +# URL is a private Azure Artifacts feed; otherwise return empty headers so nothing is sent to public hosts. +function Get-FeedAuthHeaders { + param([string]$Url) + if ($script:HasFeedAccessToken -and (Test-IsAzureArtifactsUrl -Url $Url)) { + return @{ "Authorization" = "Bearer $FeedAccessToken" } + } + return @{} +} + +# Resolve the headers once per feed type, since each feed's URL is fixed for the run. +$script:NuGetAuthHeaders = Get-FeedAuthHeaders -Url $NuGetServiceIndexUrl +$script:NpmAuthHeaders = Get-FeedAuthHeaders -Url $NpmRegistryUrl +$script:PyPiAuthHeaders = Get-FeedAuthHeaders -Url $PyPiSimpleIndexUrl +$script:MavenAuthHeaders = Get-FeedAuthHeaders -Url $MavenRepositoryUrl + +# Cache for the resolved NuGet registrations base URL so the service index is only read once. +$script:NuGetRegistrationsBaseUrl = $null + +# Resolve the NuGet registrations base URL from the service index. Prefers the gzipped SemVer 2.0.0 +# resource, matching the previously hard-coded registration5-gz-semver2 endpoint, and works against a +# private Azure Artifacts feed as well as the public api.nuget.org index. +function Get-NugetRegistrationsBaseUrl { + if ($null -ne $script:NuGetRegistrationsBaseUrl) { + return $script:NuGetRegistrationsBaseUrl + } + $index = Invoke-RestMethod -Uri $NuGetServiceIndexUrl -Method Get -Headers $script:NuGetAuthHeaders + $preferredTypes = @("RegistrationsBaseUrl/3.6.0", "RegistrationsBaseUrl/Versioned", "RegistrationsBaseUrl") + foreach ($type in $preferredTypes) { + $resource = $index.resources | Where-Object { $_.'@type' -eq $type } | Select-Object -First 1 + if ($null -ne $resource) { + $script:NuGetRegistrationsBaseUrl = $resource.'@id'.TrimEnd('/') + return $script:NuGetRegistrationsBaseUrl + } + } + throw "Could not find a RegistrationsBaseUrl resource in the NuGet service index at $NuGetServiceIndexUrl" +} + # Get the latest version of a package from NuGet function Get-LatestNugetVersion { param( [string]$packageId ) - $url = "https://api.nuget.org/v3/registration5-gz-semver2/$($packageId.ToLowerInvariant())/index.json" - $response = Invoke-RestMethod -Uri $url -Method Get + $registrationsBaseUrl = Get-NugetRegistrationsBaseUrl + $url = "$registrationsBaseUrl/$($packageId.ToLowerInvariant())/index.json" + $response = Invoke-RestMethod -Uri $url -Method Get -Headers $script:NuGetAuthHeaders $version = $response.items | Select-Object -ExpandProperty upper | ForEach-Object { [System.Management.Automation.SemanticVersion]$_ } | sort-object | Select-Object -Last 1 $version.ToString() } @@ -24,17 +102,22 @@ function Get-LatestNpmVersion { param( [string]$packageId ) - $url = "https://registry.npmjs.org/$($packageId.ToLowerInvariant())/latest" - $response = Invoke-RestMethod -Uri $url -Method Get - $response.version + # Azure Artifacts npm feeds do not implement the /{package}/latest dist-tag shortcut (they return + # 404 for it). Fetch the full packument at /{package} instead and read dist-tags.latest, which works + # against both the private feed and public registry.npmjs.org. Scoped names (e.g. + # @microsoft/kiota-abstractions) must have their '/' encoded as %2F for the packument request. + $encodedId = $packageId.ToLowerInvariant().Replace('/', '%2F') + $url = "$NpmRegistryUrl/$encodedId" + $response = Invoke-RestMethod -Uri $url -Method Get -Headers $script:NpmAuthHeaders + $response.'dist-tags'.latest } # Get the latest version of a maven package function Get-LatestMavenVersion { param( [string]$packageId ) - $url = "https://repo1.maven.org/maven2/$($packageId.Replace(":", "/").Replace(".", "/").Replace("|", "."))/maven-metadata.xml" - $response = Invoke-RestMethod -Uri $url -Method Get + $url = "$MavenRepositoryUrl/$($packageId.Replace(":", "/").Replace(".", "/").Replace("|", "."))/maven-metadata.xml" + $response = Invoke-RestMethod -Uri $url -Method Get -Headers $script:MavenAuthHeaders $response.metadata.versioning.latest } # Get the latest version of a composer package @@ -46,14 +129,71 @@ function Get-LatestComposerVersion { $response = Invoke-RestMethod -Uri $url -Method Get $response.packages.$packageId[0].version } +# Extract the version from a PyPI distribution filename ("{name}-{version}-...whl" or "{name}-{version}.tar.gz"). +# The distribution name in the filename may separate its components with '-', '_' or '.' (wheels/PEP 625 +# sdists normalize to '_', but legacy sdists can keep '-'), so we cannot assume the name is '-'-free. When the +# expected project name is known we match it tolerant of any separator and take the field that immediately +# follows it as the version (versions never contain '-'). Only when the name is unknown do we fall back to the +# second '-'-separated field. +function Get-VersionFromPypiFilename { + param( + [string]$filename, + [string]$packageId + ) + if ([string]::IsNullOrWhiteSpace($filename)) { return $null } + $stem = $filename -replace '\.(whl|egg|tar\.gz|tar\.bz2|tar\.xz|zip)$', '' + if (-not [string]::IsNullOrWhiteSpace($packageId)) { + $namePattern = ($packageId -split '[-_.]+' | Where-Object { $_ } | ForEach-Object { [regex]::Escape($_) }) -join '[-_.]' + if ($stem -match "(?i)^$namePattern-([^-]+)") { return $matches[1] } + } + $parts = $stem.Split('-') + if ($parts.Count -ge 2) { return $parts[1] } + return $null +} + +# Build a sortable key for a purely-numeric (final) release version without using [version], which is +# limited to four segments and throws on longer ones. Each dotted segment is zero-padded so an ordinal +# string sort orders the versions numerically for any number of segments. +function Get-PypiVersionSortKey { + param([string]$version) + ($version.Split('.') | ForEach-Object { '{0:D10}' -f [int]$_ }) -join '.' +} + # Get the latest version of a pypi package function Get-LatestPypiVersion { param( [string]$packageId ) - $url = "https://pypi.org/pypi/$($packageId.ToLowerInvariant())/json" - $response = Invoke-RestMethod -Uri $url -Method Get - $response.info.version + # Use the PEP 503/691 simple index (supported by both public PyPI and Azure Artifacts feeds) instead of + # the legacy pypi.org "/pypi/{pkg}/json" API, which private Azure Artifacts feeds do not expose. + # Normalize the project name per PEP 503: collapse any run of '-', '_' or '.' into a single '-' and + # lowercase. The previous per-character Replace() did not collapse consecutive separators (e.g. "a..b" + # -> "a--b", "a--b" left as-is), which produces a non-canonical name the simple index 404s on. + $normalizedId = ($packageId -replace '[-_.]+', '-').ToLowerInvariant() + $url = "$PyPiSimpleIndexUrl/$normalizedId/" + $headers = $script:PyPiAuthHeaders.Clone() + $headers["Accept"] = "application/vnd.pypi.simple.v1+json" + $response = Invoke-RestMethod -Uri $url -Method Get -Headers $headers + # The top-level "versions" array is a PEP 700 addition and is optional; PEP 691 only guarantees "files". + # Prefer "versions" when present, otherwise recover the version list from the distribution filenames so + # the resolver works against feeds that implement PEP 691 but not PEP 700. + $candidateVersions = $response.versions + if (-not $candidateVersions) { + $candidateVersions = $response.files | + ForEach-Object { Get-VersionFromPypiFilename -filename $_.filename -packageId $normalizedId } | + Where-Object { $_ } | + Select-Object -Unique + } + # Prefer the highest final (non pre-release) version to match the previous behaviour of pypi's info.version. + $stableVersions = $candidateVersions | Where-Object { $_ -match '^[0-9]+(\.[0-9]+)*$' } + if ($stableVersions) { + $stableVersions | Sort-Object -Property @{ Expression = { Get-PypiVersionSortKey -version $_ } } | Select-Object -Last 1 + } + else { + # No final release found (only pre/post/dev releases). Sort deterministically so the choice does not + # depend on the order the index happened to return, then take the highest. + $candidateVersions | Sort-Object | Select-Object -Last 1 + } } # Get the latest version of a rubygem package function Get-LatestRubygemVersion {