From d5a1b1153d2546dcc982633e68b2f55634ce2075 Mon Sep 17 00:00:00 2001 From: kl Date: Wed, 15 Jul 2026 00:44:44 +0800 Subject: [PATCH 01/53] feat: add Nexus-compatible Terraform repositories --- .github/workflows/live-compat.yml | 16 + .github/workflows/migration-e2e.yml | 21 + README.cn.md | 2 +- README.md | 2 +- .../META-INF/resources/admin/assets/admin.js | 7 +- .../resources/browse/assets/browse.js | 40 + .../resources/browse/assets/format-icons.css | 5 + .../browse/assets/formats/terraform.svg | 3 + .../META-INF/resources/browse/index.html | 10 +- compat-test/pom.xml | 10 + ...rmRepositoryBlackBoxCompatibilityTest.java | 390 ++++++++++ .../klboke/kkrepo/core/RepositoryFormat.java | 1 + .../klboke/kkrepo/core/RepositoryRecipes.java | 3 + .../security/TerraformSigningKeyMaterial.java | 46 ++ .../TerraformSigningKeyMaterialTest.java | 24 + docs/zh/dev/terraform-repository-design.md | 4 +- migration-nexus/pom.xml | 8 + .../nexus/NexusApiMigrationService.java | 124 +++- .../migration/nexus/NexusRestClient.java | 6 +- .../nexus/NexusApiMigrationServiceTest.java | 130 +++- .../jdbc/api/PersistenceStores.java | 2 + .../jdbc/api/TerraformRegistryDao.java | 73 ++ .../internal/JdbcPersistenceStoreFactory.java | 3 + .../internal/JdbcTerraformRegistryDao.java | 190 +++++ .../jdbc/contract/PersistenceApiContract.java | 49 ++ .../mysql/V30__terraform_registry.sql | 67 ++ .../MySqlV29MigrationCompatibilityTest.java | 6 +- .../postgresql/V30__terraform_registry.sql | 63 ++ .../PostgreSqlMigrationCompatibilityTest.java | 2 +- pom.xml | 23 + protocol-terraform/pom.xml | 25 + .../protocol/terraform/TerraformPath.java | 39 + .../terraform/TerraformPathParser.java | 189 +++++ .../protocol/terraform/TerraformVersions.java | 50 ++ .../terraform/TerraformPathParserTest.java | 66 ++ scripts/ci/live-compat-setup.sh | 27 + scripts/ci/run-client-e2e.sh | 117 ++- scripts/ci/run-live-compat.sh | 2 +- scripts/docker-compat/migration-e2e.sh | 151 +++- server/pom.xml | 16 + .../server/RepositoryContentController.java | 69 +- .../kkrepo/server/maven/MavenErrorAdvice.java | 5 + .../kkrepo/server/maven/MavenExceptions.java | 5 + .../RepositoryRequestMetricsFilter.java | 49 +- .../migration/NexusMigrationController.java | 9 +- .../RepositoryDataMigrationPaths.java | 4 + .../RepositoryDataMigrationWorker.java | 3 +- .../RepositoryDataMigrationWriter.java | 41 ++ .../repositories/RepositoryService.java | 53 +- .../security/RepositorySecurityFilter.java | 38 +- .../SecurityAuthenticationService.java | 19 + .../terraform/TerraformArchiveInspector.java | 179 +++++ .../terraform/TerraformAssetSupport.java | 82 +++ .../TerraformPublishLeaseManager.java | 63 ++ ...erraformRepositoryDataMigrationWriter.java | 171 +++++ .../server/terraform/TerraformService.java | 696 ++++++++++++++++++ .../terraform/TerraformSignatureVerifier.java | 68 ++ .../terraform/TerraformSigningService.java | 164 +++++ .../server/upload/ComponentUploadService.java | 64 +- .../RepositoryRequestMetricsFilterTest.java | 34 + .../NexusMigrationControllerTest.java | 5 +- .../RepositorySecurityFilterTest.java | 38 + .../TerraformArchiveInspectorTest.java | 75 ++ .../TerraformPublishLeaseManagerTest.java | 36 + ...formRepositoryDataMigrationWriterTest.java | 23 + .../terraform/TerraformServiceUrlTest.java | 31 + .../TerraformSigningServiceTest.java | 66 ++ 67 files changed, 4063 insertions(+), 39 deletions(-) create mode 100644 browse-ui/src/main/resources/META-INF/resources/browse/assets/formats/terraform.svg create mode 100644 compat-test/src/test/java/com/github/klboke/kkrepo/compat/TerraformRepositoryBlackBoxCompatibilityTest.java create mode 100644 core/src/main/java/com/github/klboke/kkrepo/core/security/TerraformSigningKeyMaterial.java create mode 100644 core/src/test/java/com/github/klboke/kkrepo/core/security/TerraformSigningKeyMaterialTest.java create mode 100644 persistence-jdbc/src/main/java/com/github/klboke/kkrepo/persistence/jdbc/api/TerraformRegistryDao.java create mode 100644 persistence-jdbc/src/main/java/com/github/klboke/kkrepo/persistence/jdbc/internal/JdbcTerraformRegistryDao.java create mode 100644 persistence-mysql/src/main/resources/db/migration/mysql/V30__terraform_registry.sql create mode 100644 persistence-postgresql/src/main/resources/db/migration/postgresql/V30__terraform_registry.sql create mode 100644 protocol-terraform/pom.xml create mode 100644 protocol-terraform/src/main/java/com/github/klboke/kkrepo/protocol/terraform/TerraformPath.java create mode 100644 protocol-terraform/src/main/java/com/github/klboke/kkrepo/protocol/terraform/TerraformPathParser.java create mode 100644 protocol-terraform/src/main/java/com/github/klboke/kkrepo/protocol/terraform/TerraformVersions.java create mode 100644 protocol-terraform/src/test/java/com/github/klboke/kkrepo/protocol/terraform/TerraformPathParserTest.java create mode 100644 server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformArchiveInspector.java create mode 100644 server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformAssetSupport.java create mode 100644 server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformPublishLeaseManager.java create mode 100644 server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformRepositoryDataMigrationWriter.java create mode 100644 server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java create mode 100644 server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformSignatureVerifier.java create mode 100644 server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformSigningService.java create mode 100644 server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformArchiveInspectorTest.java create mode 100644 server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformPublishLeaseManagerTest.java create mode 100644 server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformRepositoryDataMigrationWriterTest.java create mode 100644 server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceUrlTest.java create mode 100644 server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformSigningServiceTest.java diff --git a/.github/workflows/live-compat.yml b/.github/workflows/live-compat.yml index 82c8d511..924c456e 100644 --- a/.github/workflows/live-compat.yml +++ b/.github/workflows/live-compat.yml @@ -133,6 +133,22 @@ jobs: if: env.LIVE_COMPAT_SUITE == 'client-e2e' uses: oras-project/setup-oras@v2 + - name: Set up Terraform 0.13 and current stable + if: env.LIVE_COMPAT_SUITE == 'client-e2e' + shell: bash + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/terraform/0.13.7" "$RUNNER_TEMP/terraform/1.15.8" + for version in 0.13.7 1.15.8; do + archive="terraform_${version}_linux_amd64.zip" + curl -fsSLO "https://releases.hashicorp.com/terraform/${version}/${archive}" + curl -fsSLO "https://releases.hashicorp.com/terraform/${version}/terraform_${version}_SHA256SUMS" + grep " ${archive}$" "terraform_${version}_SHA256SUMS" | sha256sum -c - + unzip -q "$archive" -d "$RUNNER_TEMP/terraform/$version" + done + echo "TERRAFORM_013_BIN=$RUNNER_TEMP/terraform/0.13.7/terraform" >> "$GITHUB_ENV" + echo "TERRAFORM_CURRENT_BIN=$RUNNER_TEMP/terraform/1.15.8/terraform" >> "$GITHUB_ENV" + - name: Install Python client build tools if: env.LIVE_COMPAT_SUITE == 'client-e2e' run: python -m pip install --upgrade build twine diff --git a/.github/workflows/migration-e2e.yml b/.github/workflows/migration-e2e.yml index 3a9910c2..8d72212e 100644 --- a/.github/workflows/migration-e2e.yml +++ b/.github/workflows/migration-e2e.yml @@ -133,6 +133,27 @@ jobs: - name: Prepare migration fixtures run: scripts/ci/live-compat-setup.sh + - name: Set up Terraform current stable + if: matrix.nexus-version == '3.92.0' + shell: bash + run: | + set -euo pipefail + version=1.15.8 + archive="terraform_${version}_linux_amd64.zip" + mkdir -p "$RUNNER_TEMP/terraform/$version" + curl -fsSLO "https://releases.hashicorp.com/terraform/${version}/${archive}" + curl -fsSLO "https://releases.hashicorp.com/terraform/${version}/terraform_${version}_SHA256SUMS" + grep " ${archive}$" "terraform_${version}_SHA256SUMS" | sha256sum -c - + unzip -q "$archive" -d "$RUNNER_TEMP/terraform/$version" + echo "TERRAFORM_CURRENT_BIN=$RUNNER_TEMP/terraform/$version/terraform" >> "$GITHUB_ENV" + + - name: Prepare Nexus Terraform migration fixture + if: matrix.nexus-version == '3.92.0' + run: >- + mvn -B -ntp -pl compat-test -am + -Dtest=TerraformRepositoryBlackBoxCompatibilityTest#preparesNexusHostedMigrationFixtureWhenConfigured + -Dsurefire.failIfNoSpecifiedTests=false test + - name: Run Nexus metadata and repository-data migration E2E run: scripts/docker-compat/migration-e2e.sh diff --git a/README.cn.md b/README.cn.md index f8ae0f27..deaac53f 100644 --- a/README.cn.md +++ b/README.cn.md @@ -173,7 +173,7 @@ AI agent 和贡献者的开发说明见 [AGENTS.md](AGENTS.md)。 5. ohpm / HarmonyOS - 规划中，覆盖 hosted、proxy、group、导入和管理端能力（[设计说明](docs/zh/dev/ohpm-repository-design.md)） 6. Swift Package Registry 7. APT / Debian -8. Terraform Provider / Module Registry - 规划中，覆盖 hosted、proxy、group、Provider GPG 签名、Nexus 路径兼容、真实 Terraform CLI E2E 和 Nexus 迁移（[设计说明](docs/zh/dev/terraform-repository-design.md)） +8. ✅ Terraform Provider / Module Registry - hosted、proxy、group、Provider GPG 签名、Nexus 路径兼容、UI/API 上传、搜索、真实 Terraform CLI E2E 和 Nexus 迁移能力已实现（[设计说明](docs/zh/dev/terraform-repository-design.md)） 9. Conan 10. Conda diff --git a/README.md b/README.md index 57f64810..6a4a9876 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,7 @@ Repository format roadmap: 5. ohpm / HarmonyOS - Planned with hosted, proxy, group, import, and admin capabilities ([Chinese design notes](docs/zh/dev/ohpm-repository-design.md)) 6. Swift Package Registry 7. APT / Debian -8. Terraform Provider / Module Registry - Planned with hosted, proxy, group, provider GPG signing, Nexus-compatible paths, real Terraform CLI E2E, and Nexus migration ([Chinese design notes](docs/zh/dev/terraform-repository-design.md)) +8. ✅ Terraform Provider / Module Registry - Hosted, proxy, group, provider GPG signing, Nexus-compatible paths, UI/API upload, search, real Terraform CLI E2E, and Nexus migration implemented ([Chinese design notes](docs/zh/dev/terraform-repository-design.md)) 9. Conan 10. Conda diff --git a/admin-ui/src/main/resources/META-INF/resources/admin/assets/admin.js b/admin-ui/src/main/resources/META-INF/resources/admin/assets/admin.js index db04240a..4c527799 100644 --- a/admin-ui/src/main/resources/META-INF/resources/admin/assets/admin.js +++ b/admin-ui/src/main/resources/META-INF/resources/admin/assets/admin.js @@ -371,6 +371,7 @@ const FORMAT_ICON_NAMES = Object.freeze({ nuget: "nuget", rubygems: "rubygems", yum: "yum", + terraform: "terraform", raw: "raw", }); @@ -387,6 +388,7 @@ const FORMAT_DISPLAY_NAMES = Object.freeze({ nuget: "NuGet", rubygems: "RubyGems", yum: "Yum / RPM", + terraform: "Terraform", raw: "Raw", }); @@ -1718,7 +1720,7 @@ function memberCandidates() { const recipe = currentRecipe(); const format = recipe ? recipe.format : null; if (!format) return []; - const allowNestedGroups = format === "pub" || format === "composer"; + const allowNestedGroups = format === "pub" || format === "composer" || format === "terraform"; return repositories.filter((repo) => { if (repo.format !== format) return false; if (repositoryFormMode === "edit" && repo.name === editingRepositoryName) return false; @@ -1950,7 +1952,8 @@ function refreshRepositoryRemoteDefaults(recipe) { docker: "https://registry-1.docker.io/", cargo: "https://index.crates.io/", pub: "https://pub.dev/", - composer: "https://repo.packagist.org/" + composer: "https://repo.packagist.org/", + terraform: "https://registry.terraform.io/" }; remote.placeholder = defaults[recipe.format] || "https://example.com/"; if (repositoryFormMode === "create" && !remote.value.trim() && defaults[recipe.format]) { diff --git a/browse-ui/src/main/resources/META-INF/resources/browse/assets/browse.js b/browse-ui/src/main/resources/META-INF/resources/browse/assets/browse.js index 4ecdbdc7..f0a1131a 100644 --- a/browse-ui/src/main/resources/META-INF/resources/browse/assets/browse.js +++ b/browse-ui/src/main/resources/META-INF/resources/browse/assets/browse.js @@ -42,6 +42,7 @@ const SEARCH_ROUTE_FORMAT = { nuget: "nuget", pub: "pub", composer: "composer", + terraform: "terraform", pypi: "pypi", rubygems: "rubygems", yum: "yum", @@ -55,6 +56,7 @@ const FORMAT_ROUTE_SEGMENT = { nuget: "nuget", pub: "pub", composer: "composer", + terraform: "terraform", pypi: "pypi", rubygems: "rubygems", yum: "yum", @@ -145,6 +147,7 @@ function componentBrowsePath(component) { if (format === "composer" && component.kind !== "composer-package") { return name; } + if (format === "terraform") return name; return ""; } @@ -622,6 +625,7 @@ const FORMAT_ICON_NAMES = Object.freeze({ cargo: "cargo", pub: "pub", composer: "composer", + terraform: "terraform", go: "go", helm: "helm", docker: "docker", @@ -1727,6 +1731,9 @@ function hasDirectoryUsage(entry) { const reserved = new Set(["p2", "providers", "packages", "_composer"]); return (parts.length === 2 || parts.length === 3) && !reserved.has(parts[0]); } + if (repo.format === "terraform") { + return parts[0] === "v1" && (parts[1] === "modules" || parts[1] === "providers"); + } return false; } @@ -2393,6 +2400,38 @@ function renderDockerReferrers(referrers) { `; } +function terraformUsageDetail(entry) { + const parts = pathSegments(entry.path); + if (parts[0] !== "v1" || parts.length < 4) return null; + const repoUrl = repositoryUrl(currentRepository()).replace(/\/+$/, ""); + const host = window.location.host; + if (parts[1] === "modules" && parts.length >= 5) { + const [namespace, name, system] = parts.slice(2, 5); + const version = parts[5] || "1.0.0"; + return { + summaryRows: [["Format", "terraform module"], ["Namespace", namespace], ["Name", name], + ["System", system], ["Version", version]], + snippets: [ + usageSnippet("Terraform CLI config", `host "${host}" {\n services = {\n "modules.v1" = "${repoUrl}/v1/modules/"\n }\n}`), + usageSnippet("module", `module "${name}" {\n source = "${host}/${namespace}/${name}/${system}"\n version = "${version}"\n}`), + ], + }; + } + if (parts[1] === "providers" && parts.length >= 4) { + const [namespace, type] = parts.slice(2, 4); + const version = parts[4] || "1.0.0"; + return { + summaryRows: [["Format", "terraform provider"], ["Namespace", namespace], ["Type", type], + ["Version", version]], + snippets: [ + usageSnippet("Terraform CLI config", `host "${host}" {\n services = {\n "providers.v1" = "${repoUrl}/v1/providers/"\n }\n}`), + usageSnippet("required_providers", `terraform {\n required_providers {\n ${type} = {\n source = "${host}/${namespace}/${type}"\n version = "${version}"\n }\n }\n}`), + ], + }; + } + return null; +} + async function usageDetailForEntry(entry, detail = null) { const repo = currentRepository(); if (!repo) return null; @@ -2404,6 +2443,7 @@ async function usageDetailForEntry(entry, detail = null) { if (repo.format === "go") return goUsageDetail(entry); if (repo.format === "pub") return pubUsageDetail(entry); if (repo.format === "composer") return composerUsageDetail(entry, detail); + if (repo.format === "terraform") return terraformUsageDetail(entry); if (repo.format === "docker") return dockerUsageDetail(entry); return null; } diff --git a/browse-ui/src/main/resources/META-INF/resources/browse/assets/format-icons.css b/browse-ui/src/main/resources/META-INF/resources/browse/assets/format-icons.css index fc4d7022..6c0c69de 100644 --- a/browse-ui/src/main/resources/META-INF/resources/browse/assets/format-icons.css +++ b/browse-ui/src/main/resources/META-INF/resources/browse/assets/format-icons.css @@ -73,6 +73,11 @@ --format-logo: url("/browse/assets/formats/yum.svg"); } +.format-logo-terraform { + --format-color: #7b42bc; + --format-logo: url("/browse/assets/formats/terraform.svg"); +} + .format-logo-raw { --format-color: #52625e; --format-logo: url("/browse/assets/formats/raw.svg"); diff --git a/browse-ui/src/main/resources/META-INF/resources/browse/assets/formats/terraform.svg b/browse-ui/src/main/resources/META-INF/resources/browse/assets/formats/terraform.svg new file mode 100644 index 00000000..288fd2be --- /dev/null +++ b/browse-ui/src/main/resources/META-INF/resources/browse/assets/formats/terraform.svg @@ -0,0 +1,3 @@ + + + diff --git a/browse-ui/src/main/resources/META-INF/resources/browse/index.html b/browse-ui/src/main/resources/META-INF/resources/browse/index.html index 37e8b32d..9904b358 100644 --- a/browse-ui/src/main/resources/META-INF/resources/browse/index.html +++ b/browse-ui/src/main/resources/META-INF/resources/browse/index.html @@ -56,6 +56,7 @@ + @@ -113,7 +114,7 @@

Welcome

Supported repository formats

Package ecosystems supported by this kkRepo release.

- 13 formats + 14 formats diff --git a/compat-test/pom.xml b/compat-test/pom.xml index a25b1c59..4b179063 100644 --- a/compat-test/pom.xml +++ b/compat-test/pom.xml @@ -42,6 +42,16 @@ jackson-databind test + + org.bouncycastle + bcprov-jdk18on + test + + + org.bouncycastle + bcpg-jdk18on + test + org.junit.jupiter junit-jupiter diff --git a/compat-test/src/test/java/com/github/klboke/kkrepo/compat/TerraformRepositoryBlackBoxCompatibilityTest.java b/compat-test/src/test/java/com/github/klboke/kkrepo/compat/TerraformRepositoryBlackBoxCompatibilityTest.java new file mode 100644 index 00000000..cee17b97 --- /dev/null +++ b/compat-test/src/test/java/com/github/klboke/kkrepo/compat/TerraformRepositoryBlackBoxCompatibilityTest.java @@ -0,0 +1,390 @@ +package com.github.klboke.kkrepo.compat; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.ByteArrayOutputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.security.KeyPairGenerator; +import java.security.MessageDigest; +import java.security.Security; +import java.time.Duration; +import java.util.Base64; +import java.util.Date; +import java.util.HexFormat; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; +import org.bouncycastle.bcpg.ArmoredOutputStream; +import org.bouncycastle.bcpg.HashAlgorithmTags; +import org.bouncycastle.bcpg.SymmetricKeyAlgorithmTags; +import org.bouncycastle.bcpg.sig.KeyFlags; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.openpgp.PGPKeyPair; +import org.bouncycastle.openpgp.PGPKeyRingGenerator; +import org.bouncycastle.openpgp.PGPPublicKey; +import org.bouncycastle.openpgp.PGPSignature; +import org.bouncycastle.openpgp.PGPSignatureSubpacketGenerator; +import org.bouncycastle.openpgp.operator.PGPDigestCalculator; +import org.bouncycastle.openpgp.operator.jcajce.JcaKeyFingerprintCalculator; +import org.bouncycastle.openpgp.operator.jcajce.JcaPGPContentSignerBuilder; +import org.bouncycastle.openpgp.operator.jcajce.JcaPGPDigestCalculatorProviderBuilder; +import org.bouncycastle.openpgp.operator.jcajce.JcaPGPKeyPair; +import org.bouncycastle.openpgp.operator.jcajce.JcePBESecretKeyEncryptorBuilder; +import org.junit.jupiter.api.Test; + +/** Black-box contract pinned to Nexus' Terraform hosted and group HTTP behavior. */ +class TerraformRepositoryBlackBoxCompatibilityTest { + private static final HttpClient HTTP = HttpClient.newBuilder() + .connectTimeout(Duration.ofSeconds(20)).followRedirects(HttpClient.Redirect.NORMAL).build(); + private static final ObjectMapper JSON = new ObjectMapper(); + private static volatile String nexusPrivateKey; + + @Test + void preparesNexusHostedMigrationFixtureWhenConfigured() throws Exception { + Config config = Config.load(); + assumeTrue(config.configured(), + "Set NEXUS_COMPAT_BASE_URL and KKREPO_COMPAT_BASE_URL to prepare Terraform migration data"); + ensureNexus(config); + String version = "1.0." + System.currentTimeMillis(); + byte[] module = zip("main.tf", "output \"message\" { value = \"terraform migrated\" }\n" + .getBytes(StandardCharsets.UTF_8)); + byte[] provider = zip("terraform-provider-fixture_v" + version, + "#!/bin/sh\nexit 1\n".getBytes(StandardCharsets.UTF_8)); + assert2xx("Nexus migration module upload", put( + config.nexusRepository(config.hosted), + "v1/modules/kkrepo/fixture/aws/" + version + "/fixture.zip", + module, null, config.nexusAuth)); + assert2xx("Nexus migration provider upload", put( + config.nexusRepository(config.hosted), + "v1/providers/kkrepo/fixture/" + version + "/download/linux/amd64", + provider, "terraform-provider-fixture_" + version + "_linux_amd64.zip", config.nexusAuth)); + } + + @Test + void hostedModuleAndProviderContractsMatchNexusWhenConfigured() throws Exception { + Config config = Config.load(); + assumeTrue(config.configured(), + "Set NEXUS_COMPAT_BASE_URL and KKREPO_COMPAT_BASE_URL to run Terraform compatibility"); + ensureNexus(config); + ensureKkRepo(config); + + String version = "1.0." + System.currentTimeMillis(); + byte[] module = zip("main.tf", "output \"message\" { value = \"terraform compat\" }\n".getBytes(StandardCharsets.UTF_8)); + byte[] provider = zip("terraform-provider-fixture_v" + version, + "#!/bin/sh\nexit 1\n".getBytes(StandardCharsets.UTF_8)); + String modulePath = "v1/modules/kkrepo/fixture/aws/" + version + "/fixture.zip"; + String providerPath = "v1/providers/kkrepo/fixture/" + version + "/download/linux/amd64"; + String providerFilename = "terraform-provider-fixture_" + version + "_linux_amd64.zip"; + + Exchange nexusModulePut = put( + config.nexusRepository(config.hosted), modulePath, module, null, config.nexusAuth); + Exchange kkrepoModulePut = put( + config.kkrepoRepository(config.hosted), modulePath, module, null, config.kkrepoAuth); + assertEquals(nexusModulePut.status, kkrepoModulePut.status, "module upload status"); + assert2xx("Nexus module upload", nexusModulePut); + + Exchange nexusVersions = get(config.nexusRepository(config.group), + "v1/modules/kkrepo/fixture/aws/versions", config.nexusAuth); + Exchange kkrepoVersions = get(config.kkrepoRepository(config.group), + "v1/modules/kkrepo/fixture/aws/versions", config.kkrepoAuth); + assertEquals(nexusVersions.status, kkrepoVersions.status, "module versions status"); + assertTrue(nexusVersions.text().contains(version)); + assertTrue(kkrepoVersions.text().contains(version)); + + Exchange nexusDownload = get(config.nexusRepository(config.group), + "v1/modules/kkrepo/fixture/aws/" + version + "/download", config.nexusAuth); + Exchange kkrepoDownload = get(config.kkrepoRepository(config.group), + "v1/modules/kkrepo/fixture/aws/" + version + "/download", config.kkrepoAuth); + assertEquals(204, nexusDownload.status); + assertEquals(nexusDownload.status, kkrepoDownload.status, "module download status"); + assertTrue(nexusDownload.header("x-terraform-get").contains("/repository/" + config.group + "/")); + assertTrue(kkrepoDownload.header("x-terraform-get").contains("/repository/" + config.group + "/")); + String nexusModuleUrl = config.resolve( + nexusDownload.header("x-terraform-get"), config.nexusRepository(config.group)); + String kkrepoModuleUrl = config.resolve( + kkrepoDownload.header("x-terraform-get"), config.kkrepoRepository(config.group)); + assertArrayEquals(module, getAbsolute(nexusModuleUrl, config.nexusAuth).body); + assertArrayEquals(module, getAbsolute(kkrepoModuleUrl, config.kkrepoAuth).body); + assertDownloadValidatorsAndRangeParity( + nexusModuleUrl, config.nexusAuth, kkrepoModuleUrl, config.kkrepoAuth, + nexusModuleUrl.replace("/" + config.group + "/", "/" + config.hosted + "/"), + kkrepoModuleUrl.replace("/" + config.group + "/", "/" + config.hosted + "/"), + module); + + Exchange nexusProviderPut = put(config.nexusRepository(config.hosted), providerPath, provider, + providerFilename, config.nexusAuth); + Exchange kkrepoProviderPut = put(config.kkrepoRepository(config.hosted), providerPath, provider, + providerFilename, config.kkrepoAuth); + assertEquals(nexusProviderPut.status, kkrepoProviderPut.status, "provider upload status"); + assert2xx("Nexus provider upload", nexusProviderPut); + + validateProvider(config, config.nexusRepository(config.group), config.nexusAuth, + providerPath, providerFilename, provider); + validateProvider(config, config.kkrepoRepository(config.group), config.kkrepoAuth, + providerPath, providerFilename, provider); + + Exchange badNexus = get(config.nexusRepository(config.group), + "v1/providers/kkrepo/fixture/" + version + "/download/plan9/amd64", config.nexusAuth); + Exchange badKkRepo = get(config.kkrepoRepository(config.group), + "v1/providers/kkrepo/fixture/" + version + "/download/plan9/amd64", config.kkrepoAuth); + assertEquals(badNexus.status, badKkRepo.status, "missing platform status"); + assertEquals(404, badKkRepo.status); + } + + private static void validateProvider( + Config config, String repository, String authorization, String path, String filename, + byte[] expectedArchive) throws Exception { + Exchange metadata = get(repository, path, authorization); + assertEquals(200, metadata.status, metadata.text()); + Map body = JSON.readValue(metadata.body, new TypeReference<>() {}); + assertEquals(filename, body.get("filename")); + String shasum = body.get("shasum").toString(); + assertEquals(sha256(expectedArchive), shasum); + byte[] archive = getAbsolute( + config.resolve(body.get("download_url").toString(), repository), authorization).body; + assertArrayEquals(expectedArchive, archive); + String sums = getAbsolute( + config.resolve(body.get("shasums_url").toString(), repository), authorization).text(); + assertTrue(sums.lines().anyMatch(line -> line.trim().equalsIgnoreCase(shasum + " " + filename))); + Exchange signature = getAbsolute(config.resolve( + body.get("shasums_signature_url").toString(), repository), authorization); + assert2xx("provider signature", signature); + assertTrue(signature.body.length > 32); + @SuppressWarnings("unchecked") + Map signingKeys = (Map) body.get("signing_keys"); + assertFalse(signingKeys.isEmpty()); + assertTrue(signingKeys.toString().contains("BEGIN PGP PUBLIC KEY BLOCK")); + } + + private static void assertDownloadValidatorsAndRangeParity( + String nexusUrl, String nexusAuthorization, String kkrepoUrl, String kkrepoAuthorization, + String nexusHostedUrl, String kkrepoHostedUrl, byte[] expected) throws Exception { + Exchange nexusHead = send(HttpRequest.newBuilder(URI.create(nexusUrl)) + .header("Authorization", nexusAuthorization).method("HEAD", HttpRequest.BodyPublishers.noBody())); + Exchange kkrepoHead = send(HttpRequest.newBuilder(URI.create(kkrepoUrl)) + .header("Authorization", kkrepoAuthorization).method("HEAD", HttpRequest.BodyPublishers.noBody())); + assertEquals(404, nexusHead.status, "Nexus 3.92 Terraform group does not dispatch archive HEAD"); + assertEquals(nexusHead.status, kkrepoHead.status, "group archive HEAD status"); + + Exchange nexusRange = send(HttpRequest.newBuilder(URI.create(nexusUrl)) + .header("Authorization", nexusAuthorization).header("Range", "bytes=0-3").GET()); + Exchange kkrepoRange = send(HttpRequest.newBuilder(URI.create(kkrepoUrl)) + .header("Authorization", kkrepoAuthorization).header("Range", "bytes=0-3").GET()); + assertEquals(nexusRange.status, kkrepoRange.status, "archive Range status"); + assertEquals(200, kkrepoRange.status, "Nexus 3.92 Terraform facet ignores Range"); + assertArrayEquals(expected, nexusRange.body); + assertArrayEquals(expected, kkrepoRange.body); + + Exchange nexusHostedHead = send(HttpRequest.newBuilder(URI.create(nexusHostedUrl)) + .header("Authorization", nexusAuthorization).method("HEAD", HttpRequest.BodyPublishers.noBody())); + Exchange kkrepoHostedHead = send(HttpRequest.newBuilder(URI.create(kkrepoHostedUrl)) + .header("Authorization", kkrepoAuthorization).method("HEAD", HttpRequest.BodyPublishers.noBody())); + assertEquals(nexusHostedHead.status, kkrepoHostedHead.status, "hosted archive HEAD status"); + assertEquals(Integer.toString(expected.length), nexusHostedHead.header("content-length")); + assertEquals(Integer.toString(expected.length), kkrepoHostedHead.header("content-length")); + assertFalse(nexusHostedHead.header("etag").isBlank()); + assertFalse(kkrepoHostedHead.header("etag").isBlank()); + assertFalse(nexusHostedHead.header("last-modified").isBlank()); + assertFalse(kkrepoHostedHead.header("last-modified").isBlank()); + + Exchange nexusConditional = send(HttpRequest.newBuilder(URI.create(nexusHostedUrl)) + .header("Authorization", nexusAuthorization) + .header("If-None-Match", nexusHostedHead.header("etag")).GET()); + Exchange kkrepoConditional = send(HttpRequest.newBuilder(URI.create(kkrepoHostedUrl)) + .header("Authorization", kkrepoAuthorization) + .header("If-None-Match", kkrepoHostedHead.header("etag")).GET()); + assertEquals(304, nexusConditional.status); + assertEquals(nexusConditional.status, kkrepoConditional.status, "archive conditional GET status"); + } + + private static void ensureNexus(Config config) throws Exception { + String repositories = send(config.nexusAdmin("/service/rest/v1/repositories").GET()).text(); + if (!repositories.contains("\"name\" : \"" + config.hosted + "\"")) { + assert2xx("create Nexus Terraform hosted", send(config.nexusAdmin( + "/service/rest/v1/repositories/terraform/hosted") + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(JSON.writeValueAsString(Map.of( + "name", config.hosted, + "online", true, + "storage", Map.of( + "blobStoreName", "default", + "strictContentTypeValidation", true, + "writePolicy", "ALLOW_ONCE"), + "terraformSigning", Map.of( + "keypair", nexusPrivateKey(), + "passphrase", ""))))))); + } + repositories = send(config.nexusAdmin("/service/rest/v1/repositories").GET()).text(); + if (!repositories.contains("\"name\" : \"" + config.proxy + "\"")) { + assert2xx("create Nexus Terraform proxy", send(config.nexusAdmin( + "/service/rest/v1/repositories/terraform/proxy") + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(""" + {"name":"%s","online":true,"storage":{"blobStoreName":"default","strictContentTypeValidation":true},"proxy":{"remoteUrl":"https://registry.terraform.io/","contentMaxAge":1440,"metadataMaxAge":1440},"negativeCache":{"enabled":true,"timeToLive":1440},"httpClient":{"blocked":false,"autoBlock":true}} + """.formatted(config.proxy))))); + } + repositories = send(config.nexusAdmin("/service/rest/v1/repositories").GET()).text(); + if (!repositories.contains("\"name\" : \"" + config.group + "\"")) { + assert2xx("create Nexus Terraform group", send(config.nexusAdmin( + "/service/rest/v1/repositories/terraform/group") + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(""" + {"name":"%s","online":true,"storage":{"blobStoreName":"default","strictContentTypeValidation":true},"group":{"memberNames":["%s","%s"]}} + """.formatted(config.group, config.hosted, config.proxy))))); + } + } + + private static void ensureKkRepo(Config config) throws Exception { + String repositories = send(config.kkrepoAdmin("/internal/repositories").GET()).text(); + if (!repositories.contains("\"name\":\"" + config.hosted + "\"")) { + assert2xx("create kkrepo Terraform hosted", send(config.kkrepoAdmin("/internal/repositories") + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(""" + {"name":"%s","recipe":"terraform-hosted","online":true,"blobStoreName":"default","strictContentTypeValidation":true,"hosted":{"writePolicy":"ALLOW_ONCE"}} + """.formatted(config.hosted))))); + } + repositories = send(config.kkrepoAdmin("/internal/repositories").GET()).text(); + if (!repositories.contains("\"name\":\"" + config.proxy + "\"")) { + assert2xx("create kkrepo Terraform proxy", send(config.kkrepoAdmin("/internal/repositories") + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(""" + {"name":"%s","recipe":"terraform-proxy","online":true,"blobStoreName":"default","strictContentTypeValidation":true,"proxy":{"remoteUrl":"https://registry.terraform.io/","contentMaxAgeMinutes":1440,"metadataMaxAgeMinutes":1440,"autoBlock":true}} + """.formatted(config.proxy))))); + } + repositories = send(config.kkrepoAdmin("/internal/repositories").GET()).text(); + if (!repositories.contains("\"name\":\"" + config.group + "\"")) { + assert2xx("create kkrepo Terraform group", send(config.kkrepoAdmin("/internal/repositories") + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(""" + {"name":"%s","recipe":"terraform-group","online":true,"blobStoreName":"default","strictContentTypeValidation":true,"group":{"memberNames":["%s","%s"]}} + """.formatted(config.group, config.hosted, config.proxy))))); + } + } + + private static Exchange put( + String repository, String path, byte[] body, String filename, String authorization) throws Exception { + HttpRequest.Builder request = HttpRequest.newBuilder(URI.create(repository + "/" + path)) + .header("Content-Type", "application/zip") + .header("Authorization", authorization); + if (filename != null) request.header("Content-Disposition", "attachment; filename=\"" + filename + "\""); + return send(request.PUT(HttpRequest.BodyPublishers.ofByteArray(body))); + } + + private static Exchange get(String repository, String path, String authorization) throws Exception { + return send(HttpRequest.newBuilder(URI.create(repository + "/" + path)) + .header("Authorization", authorization).GET()); + } + + private static Exchange getAbsolute(String url, String authorization) throws Exception { + return send(HttpRequest.newBuilder(URI.create(url)).header("Authorization", authorization).GET()); + } + + private static Exchange send(HttpRequest.Builder request) throws Exception { + HttpResponse response = HTTP.send(request.timeout(Duration.ofSeconds(120)).build(), + HttpResponse.BodyHandlers.ofByteArray()); + return new Exchange(response.statusCode(), response.body(), response.headers().map()); + } + + private static byte[] zip(String name, byte[] content) throws Exception { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ZipOutputStream zip = new ZipOutputStream(bytes)) { + zip.putNextEntry(new ZipEntry(name)); + zip.write(content); + zip.closeEntry(); + } + return bytes.toByteArray(); + } + + private static String sha256(byte[] value) throws Exception { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(value)); + } + + private static synchronized String nexusPrivateKey() throws Exception { + if (nexusPrivateKey != null) return nexusPrivateKey; + if (Security.getProvider(BouncyCastleProvider.PROVIDER_NAME) == null) { + Security.addProvider(new BouncyCastleProvider()); + } + KeyPairGenerator generator = KeyPairGenerator.getInstance("RSA", BouncyCastleProvider.PROVIDER_NAME); + generator.initialize(2048); + PGPKeyPair pair = new JcaPGPKeyPair( + PGPPublicKey.RSA_SIGN, generator.generateKeyPair(), new Date()); + PGPDigestCalculator sha1 = new JcaPGPDigestCalculatorProviderBuilder() + .setProvider(BouncyCastleProvider.PROVIDER_NAME).build().get(HashAlgorithmTags.SHA1); + PGPSignatureSubpacketGenerator certification = new PGPSignatureSubpacketGenerator(); + certification.setKeyFlags(false, KeyFlags.CERTIFY_OTHER | KeyFlags.SIGN_DATA); + PGPKeyRingGenerator rings = new PGPKeyRingGenerator( + PGPSignature.POSITIVE_CERTIFICATION, + pair, + "kkrepo Terraform compatibility ", + sha1, + certification.generate(), + null, + new JcaPGPContentSignerBuilder(pair.getPublicKey().getAlgorithm(), HashAlgorithmTags.SHA256) + .setProvider(BouncyCastleProvider.PROVIDER_NAME), + new JcePBESecretKeyEncryptorBuilder(SymmetricKeyAlgorithmTags.AES_256, sha1) + .setProvider(BouncyCastleProvider.PROVIDER_NAME).build(new char[0])); + ByteArrayOutputStream armored = new ByteArrayOutputStream(); + try (ArmoredOutputStream output = new ArmoredOutputStream(armored)) { + rings.generateSecretKeyRing().encode(output); + } + nexusPrivateKey = armored.toString(StandardCharsets.UTF_8); + // Exercise the same parser used by registry clients before sending the fixture key to Nexus. + new org.bouncycastle.openpgp.PGPSecretKeyRingCollection( + org.bouncycastle.openpgp.PGPUtil.getDecoderStream( + new java.io.ByteArrayInputStream(nexusPrivateKey.getBytes(StandardCharsets.UTF_8))), + new JcaKeyFingerprintCalculator()); + return nexusPrivateKey; + } + + private static void assert2xx(String label, Exchange exchange) { + assertTrue(exchange.status >= 200 && exchange.status < 300, + label + " expected 2xx but got " + exchange.status + " body=" + exchange.text()); + } + + private record Exchange(int status, byte[] body, Map> headers) { + String text() { return new String(body, StandardCharsets.UTF_8); } + String header(String name) { + return headers.entrySet().stream().filter(entry -> entry.getKey().equalsIgnoreCase(name)) + .flatMap(entry -> entry.getValue().stream()).findFirst().orElse(""); + } + } + + private record Config( + String nexusBase, String kkrepoBase, String nexusAuth, String kkrepoAuth, + String hosted, String proxy, String group) { + static Config load() { + return new Config( + CompatDefaults.nexusBaseUrl().orElse(""), CompatDefaults.nexusPlusBaseUrl().orElse(""), + basic(CompatDefaults.nexusUsername().orElse(""), CompatDefaults.nexusPassword().orElse("")), + basic(CompatDefaults.nexusPlusUsername().orElse(""), CompatDefaults.nexusPlusPassword().orElse("")), + "terraform-compat-hosted", "terraform-compat-proxy", "terraform-compat-group"); + } + + boolean configured() { return !nexusBase.isBlank() && !kkrepoBase.isBlank(); } + String nexusRepository(String repository) { return nexusBase + "/repository/" + repository; } + String kkrepoRepository(String repository) { return kkrepoBase + "/repository/" + repository; } + HttpRequest.Builder nexusAdmin(String path) { + return HttpRequest.newBuilder(URI.create(nexusBase + path)).header("Authorization", nexusAuth); + } + HttpRequest.Builder kkrepoAdmin(String path) { + return HttpRequest.newBuilder(URI.create(kkrepoBase + path)).header("Authorization", kkrepoAuth); + } + String resolve(String value, String repositoryBase) { + if (value.startsWith("http://") || value.startsWith("https://")) return value; + return URI.create(repositoryBase + "/").resolve(value).toString(); + } + private static String basic(String user, String password) { + return "Basic " + Base64.getEncoder().encodeToString((user + ":" + password).getBytes(StandardCharsets.UTF_8)); + } + } +} diff --git a/core/src/main/java/com/github/klboke/kkrepo/core/RepositoryFormat.java b/core/src/main/java/com/github/klboke/kkrepo/core/RepositoryFormat.java index 55572079..266ee53e 100644 --- a/core/src/main/java/com/github/klboke/kkrepo/core/RepositoryFormat.java +++ b/core/src/main/java/com/github/klboke/kkrepo/core/RepositoryFormat.java @@ -17,6 +17,7 @@ public enum RepositoryFormat { NUGET, RUBYGEMS, YUM, + TERRAFORM, RAW; @JsonCreator diff --git a/core/src/main/java/com/github/klboke/kkrepo/core/RepositoryRecipes.java b/core/src/main/java/com/github/klboke/kkrepo/core/RepositoryRecipes.java index b6fa708c..89e3977e 100644 --- a/core/src/main/java/com/github/klboke/kkrepo/core/RepositoryRecipes.java +++ b/core/src/main/java/com/github/klboke/kkrepo/core/RepositoryRecipes.java @@ -47,6 +47,9 @@ public final class RepositoryRecipes { new RepositoryRecipe("yum-hosted", RepositoryFormat.YUM, RepositoryType.HOSTED), new RepositoryRecipe("yum-proxy", RepositoryFormat.YUM, RepositoryType.PROXY), new RepositoryRecipe("yum-group", RepositoryFormat.YUM, RepositoryType.GROUP), + new RepositoryRecipe("terraform-hosted", RepositoryFormat.TERRAFORM, RepositoryType.HOSTED), + new RepositoryRecipe("terraform-proxy", RepositoryFormat.TERRAFORM, RepositoryType.PROXY), + new RepositoryRecipe("terraform-group", RepositoryFormat.TERRAFORM, RepositoryType.GROUP), new RepositoryRecipe("raw-hosted", RepositoryFormat.RAW, RepositoryType.HOSTED), new RepositoryRecipe("raw-proxy", RepositoryFormat.RAW, RepositoryType.PROXY), new RepositoryRecipe("raw-group", RepositoryFormat.RAW, RepositoryType.GROUP)); diff --git a/core/src/main/java/com/github/klboke/kkrepo/core/security/TerraformSigningKeyMaterial.java b/core/src/main/java/com/github/klboke/kkrepo/core/security/TerraformSigningKeyMaterial.java new file mode 100644 index 00000000..e0a3189e --- /dev/null +++ b/core/src/main/java/com/github/klboke/kkrepo/core/security/TerraformSigningKeyMaterial.java @@ -0,0 +1,46 @@ +package com.github.klboke.kkrepo.core.security; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +/** Versioned plaintext envelope stored only inside {@link SecretCipher} ciphertext. */ +public final class TerraformSigningKeyMaterial { + private static final String HEADER = "KKREPO_TERRAFORM_SIGNING_KEY_V1"; + + private TerraformSigningKeyMaterial() { + } + + public static String encode(String privateKeyArmor, String passphrase) { + if (privateKeyArmor == null || privateKeyArmor.isBlank()) { + throw new IllegalArgumentException("Terraform signing private key is required"); + } + String encodedPassphrase = Base64.getEncoder().encodeToString( + (passphrase == null ? "" : passphrase).getBytes(StandardCharsets.UTF_8)); + return HEADER + "\n" + encodedPassphrase + "\n" + privateKeyArmor; + } + + public static Material decode(String plaintext) { + if (plaintext == null || plaintext.isBlank()) { + throw new IllegalArgumentException("Terraform signing private key is required"); + } + String prefix = HEADER + "\n"; + if (!plaintext.startsWith(prefix)) { + // Backward-compatible with keys created before passphrase migration support. + return new Material(plaintext, ""); + } + int separator = plaintext.indexOf('\n', prefix.length()); + if (separator < 0) { + throw new IllegalArgumentException("Invalid Terraform signing key envelope"); + } + String passphrase = new String(Base64.getDecoder().decode( + plaintext.substring(prefix.length(), separator)), StandardCharsets.UTF_8); + String privateKey = plaintext.substring(separator + 1); + if (privateKey.isBlank()) { + throw new IllegalArgumentException("Invalid Terraform signing key envelope"); + } + return new Material(privateKey, passphrase); + } + + public record Material(String privateKeyArmor, String passphrase) { + } +} diff --git a/core/src/test/java/com/github/klboke/kkrepo/core/security/TerraformSigningKeyMaterialTest.java b/core/src/test/java/com/github/klboke/kkrepo/core/security/TerraformSigningKeyMaterialTest.java new file mode 100644 index 00000000..3d50d8cb --- /dev/null +++ b/core/src/test/java/com/github/klboke/kkrepo/core/security/TerraformSigningKeyMaterialTest.java @@ -0,0 +1,24 @@ +package com.github.klboke.kkrepo.core.security; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class TerraformSigningKeyMaterialTest { + @Test + void roundTripsArmoredKeyAndPassphrase() { + var decoded = TerraformSigningKeyMaterial.decode( + TerraformSigningKeyMaterial.encode("-----BEGIN PGP PRIVATE KEY BLOCK-----", "p@ssphrase")); + + assertEquals("-----BEGIN PGP PRIVATE KEY BLOCK-----", decoded.privateKeyArmor()); + assertEquals("p@ssphrase", decoded.passphrase()); + } + + @Test + void decodesLegacyArmorWithEmptyPassphrase() { + var decoded = TerraformSigningKeyMaterial.decode("legacy-private-armor"); + + assertEquals("legacy-private-armor", decoded.privateKeyArmor()); + assertEquals("", decoded.passphrase()); + } +} diff --git a/docs/zh/dev/terraform-repository-design.md b/docs/zh/dev/terraform-repository-design.md index 8eebcc6d..34391760 100644 --- a/docs/zh/dev/terraform-repository-design.md +++ b/docs/zh/dev/terraform-repository-design.md @@ -4,7 +4,7 @@ ## 当前支持状态 -Terraform 仓库处于路线图设计阶段。当前代码尚未包含 `RepositoryFormat.TERRAFORM`、`terraform-hosted`、`terraform-proxy`、`terraform-group` recipe 或 `protocol-terraform` 模块。 +截至 2026-07-14，Terraform 第一阶段仓库能力已经落地：代码包含 `RepositoryFormat.TERRAFORM`、`terraform-hosted`、`terraform-proxy`、`terraform-group` recipe、独立 `protocol-terraform` 模块、共享 Provider revision/签名/source binding 数据模型，以及服务端、UI/API 上传、Browse/Search、迁移和兼容性测试入口。 完整实现目标覆盖： @@ -52,7 +52,7 @@ Terraform 仓库处于路线图设计阶段。当前代码尚未包含 `Reposito - Provider package 只接受 `.zip`；校验 `Content-Disposition`、URL identity、文件名、zip 结构、平台 identity 和 SHA256。 - 为 hosted Provider 生成 SHA256SUMS、detached GPG signature 和 registry metadata；签名私钥按 secret 保存，不进入日志、公开 metadata 或普通 repository JSON。 - 支持同一 Provider version 增量上传多个平台，并保证 versions/platforms/checksum/signature 一致可见。 - - 支持 `HEAD`、Range、ETag、Last-Modified 和 conditional GET；具体状态/header 与 Nexus 参考行为对齐。 + - hosted archive 支持 `HEAD`、ETag、Last-Modified 和 conditional GET；group archive 的 `HEAD` 保持 Nexus 3.92 当前的 `404`。Range 请求保持 Nexus 3.92 当前的 `200` 全量响应语义（若后续 Nexus 改变则由兼容测试驱动调整）。 2. Terraform proxy - 新增 `terraform-proxy` recipe；每个 proxy 只配置一个上游 registry base URL。 diff --git a/migration-nexus/pom.xml b/migration-nexus/pom.xml index 3874fc4b..e72e45c9 100644 --- a/migration-nexus/pom.xml +++ b/migration-nexus/pom.xml @@ -24,6 +24,14 @@ com.fasterxml.jackson.core jackson-databind + + org.bouncycastle + bcprov-jdk18on + + + org.bouncycastle + bcpg-jdk18on + com.mysql mysql-connector-j diff --git a/migration-nexus/src/main/java/com/github/klboke/kkrepo/migration/nexus/NexusApiMigrationService.java b/migration-nexus/src/main/java/com/github/klboke/kkrepo/migration/nexus/NexusApiMigrationService.java index 19c4324d..aa3ec140 100644 --- a/migration-nexus/src/main/java/com/github/klboke/kkrepo/migration/nexus/NexusApiMigrationService.java +++ b/migration-nexus/src/main/java/com/github/klboke/kkrepo/migration/nexus/NexusApiMigrationService.java @@ -1,8 +1,12 @@ package com.github.klboke.kkrepo.migration.nexus; import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.klboke.kkrepo.core.RepositoryFormat; import com.github.klboke.kkrepo.core.RepositoryRecipe; import com.github.klboke.kkrepo.core.RepositoryType; +import com.github.klboke.kkrepo.core.security.EncryptionSecrets; +import com.github.klboke.kkrepo.core.security.SecretCipher; +import com.github.klboke.kkrepo.core.security.TerraformSigningKeyMaterial; import com.github.klboke.kkrepo.migration.nexus.NexusRestClient.NexusInventory; import com.github.klboke.kkrepo.migration.nexus.NexusRestClient.RepositoryDocument; import com.github.klboke.kkrepo.migration.nexus.MigrationPlanBuilder.MigrationScope; @@ -22,8 +26,10 @@ import com.github.klboke.kkrepo.persistence.jdbc.api.MigrationJobDao; import com.github.klboke.kkrepo.persistence.jdbc.api.RepositoryDao; import com.github.klboke.kkrepo.persistence.jdbc.api.SecurityDao; +import com.github.klboke.kkrepo.persistence.jdbc.api.TerraformRegistryDao; import com.github.klboke.kkrepo.persistence.jdbc.api.model.BlobStoreRecord; import com.github.klboke.kkrepo.persistence.jdbc.api.model.RepositoryRecord; +import java.security.Security; import java.util.ArrayList; import java.util.Comparator; import java.util.LinkedHashMap; @@ -32,6 +38,14 @@ import java.util.Locale; import java.util.Map; import java.util.Optional; +import org.bouncycastle.bcpg.ArmoredOutputStream; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.openpgp.PGPSecretKey; +import org.bouncycastle.openpgp.PGPSecretKeyRing; +import org.bouncycastle.openpgp.PGPSecretKeyRingCollection; +import org.bouncycastle.openpgp.PGPUtil; +import org.bouncycastle.openpgp.operator.jcajce.JcaKeyFingerprintCalculator; +import org.bouncycastle.openpgp.operator.jcajce.JcePBESecretKeyDecryptorBuilder; import org.springframework.transaction.annotation.Transactional; public class NexusApiMigrationService { @@ -43,6 +57,7 @@ public class NexusApiMigrationService { private final SecurityDao securityDao; private final MigrationJobDao migrationJobDao; private final NexusSecurityMigrationWriter securityWriter; + private final TerraformRegistryDao terraformRegistry; public NexusApiMigrationService( ObjectMapper objectMapper, @@ -51,12 +66,24 @@ public NexusApiMigrationService( SecurityDao securityDao, MigrationJobDao migrationJobDao, NexusSecurityMigrationWriter securityWriter) { + this(objectMapper, blobStoreDao, repositoryDao, securityDao, migrationJobDao, securityWriter, null); + } + + public NexusApiMigrationService( + ObjectMapper objectMapper, + BlobStoreDao blobStoreDao, + RepositoryDao repositoryDao, + SecurityDao securityDao, + MigrationJobDao migrationJobDao, + NexusSecurityMigrationWriter securityWriter, + TerraformRegistryDao terraformRegistry) { this.objectMapper = objectMapper; this.blobStoreDao = blobStoreDao; this.repositoryDao = repositoryDao; this.securityDao = securityDao; this.migrationJobDao = migrationJobDao; this.securityWriter = securityWriter; + this.terraformRegistry = terraformRegistry; } public NexusMigrationPreflight preflight(NexusMigrationRequest request) @@ -316,7 +343,9 @@ ConfigMigrationCounts migrateConfig(NexusInventory inventory, NexusMigrationRequ .toList(); for (RepositoryDocument document : supported) { RepositoryRecord record = repositoryRecord(document, targetBlobStores); - migrated.put(record.name(), repositoryDao.upsertByName(record)); + RepositoryRecord stored = repositoryDao.upsertByName(record); + migrated.put(record.name(), stored); + migrateTerraformSigningKey(document, stored); } for (RepositoryDocument document : supported) { if (repositoryTypeEnum(document) != RepositoryType.GROUP) { @@ -409,7 +438,8 @@ private RepositoryRecord repositoryRecord( LinkedHashMap attributes = new LinkedHashMap<>(); attributes.put("recipe", recipe.name()); attributes.put("source", "nexus-migration"); - attributes.put("sourceRepository", document.detail().isEmpty() ? document.summary() : document.detail()); + attributes.put("sourceRepository", redactSecrets( + document.detail().isEmpty() ? document.summary() : document.detail())); if (recipe.type() == RepositoryType.PROXY) { attributes.put("proxy", proxyAttributes(document, recipe)); } @@ -436,6 +466,96 @@ private RepositoryRecord repositoryRecord( Map.copyOf(attributes)); } + private void migrateTerraformSigningKey(RepositoryDocument document, RepositoryRecord repository) { + if (terraformRegistry == null + || repository.id() == null + || repository.format() != RepositoryFormat.TERRAFORM + || repository.type() != RepositoryType.HOSTED + || terraformRegistry.findActiveSigningKey(repository.id()).isPresent()) { + return; + } + Object raw = value(document, "terraformSigning"); + if (!(raw instanceof Map signing)) { + return; + } + String privateArmor = string(signing.get("keypair")); + if (privateArmor == null || privateArmor.isBlank()) { + return; + } + String passphrase = defaultString(string(signing.get("passphrase")), ""); + try { + if (Security.getProvider(BouncyCastleProvider.PROVIDER_NAME) == null) { + Security.addProvider(new BouncyCastleProvider()); + } + PGPSecretKeyRingCollection rings = new PGPSecretKeyRingCollection( + PGPUtil.getDecoderStream(new java.io.ByteArrayInputStream( + privateArmor.getBytes(java.nio.charset.StandardCharsets.UTF_8))), + new JcaKeyFingerprintCalculator()); + PGPSecretKeyRing selectedRing = null; + PGPSecretKey signingKey = null; + java.util.Iterator ringIterator = rings.getKeyRings(); + while (ringIterator.hasNext() && signingKey == null) { + PGPSecretKeyRing candidateRing = ringIterator.next(); + java.util.Iterator keys = candidateRing.getSecretKeys(); + while (keys.hasNext()) { + PGPSecretKey candidate = keys.next(); + if (candidate.isSigningKey()) { + candidate.extractPrivateKey(new JcePBESecretKeyDecryptorBuilder() + .setProvider(BouncyCastleProvider.PROVIDER_NAME) + .build(passphrase.toCharArray())); + selectedRing = candidateRing; + signingKey = candidate; + break; + } + } + } + if (selectedRing == null || signingKey == null) { + throw new IllegalArgumentException( + "Nexus Terraform signing key cannot sign for repository " + repository.name()); + } + java.io.ByteArrayOutputStream publicArmor = new java.io.ByteArrayOutputStream(); + try (ArmoredOutputStream armor = new ArmoredOutputStream(publicArmor)) { + selectedRing.toCertificate().encode(armor); + } + String encrypted = new SecretCipher(EncryptionSecrets.credentialSecret()).encrypt( + TerraformSigningKeyMaterial.encode(privateArmor, passphrase)); + terraformRegistry.insertSigningKey(new TerraformRegistryDao.SigningKey( + repository.id(), + 1, + String.format("%016X", signingKey.getKeyID()), + encrypted, + publicArmor.toString(java.nio.charset.StandardCharsets.UTF_8), + java.time.Instant.now())); + } catch (Exception e) { + throw new IllegalArgumentException( + "Failed importing Nexus Terraform signing key for repository " + repository.name(), e); + } + } + + private static Object redactSecrets(Object value) { + if (value instanceof Map source) { + LinkedHashMap sanitized = new LinkedHashMap<>(); + source.forEach((key, child) -> { + String name = String.valueOf(key); + String lower = name.toLowerCase(Locale.ROOT); + boolean sensitive = lower.contains("password") + || lower.contains("passphrase") + || lower.contains("secret") + || lower.contains("credential") + || lower.contains("bearer") + || lower.contains("token") + || lower.contains("keypair") + || lower.contains("privatekey"); + sanitized.put(name, sensitive ? "" : redactSecrets(child)); + }); + return java.util.Collections.unmodifiableMap(sanitized); + } + if (value instanceof List values) { + return values.stream().map(NexusApiMigrationService::redactSecrets).toList(); + } + return value; + } + private NexusSecurityMigrationResult migrateSecurity( NexusSecurityExport export, boolean dryRun) { diff --git a/migration-nexus/src/main/java/com/github/klboke/kkrepo/migration/nexus/NexusRestClient.java b/migration-nexus/src/main/java/com/github/klboke/kkrepo/migration/nexus/NexusRestClient.java index 15ee300f..1d220dff 100644 --- a/migration-nexus/src/main/java/com/github/klboke/kkrepo/migration/nexus/NexusRestClient.java +++ b/migration-nexus/src/main/java/com/github/klboke/kkrepo/migration/nexus/NexusRestClient.java @@ -404,7 +404,8 @@ public class NexusRestClient { composer: 'COMPOSER', yum: 'YUM', raw: 'RAW', - cargo: 'CARGO' + cargo: 'CARGO', + terraform: 'TERRAFORM' ] return prefixes[format] } @@ -811,7 +812,8 @@ public class NexusRestClient { composer: 'COMPOSER', yum: 'YUM', raw: 'RAW', - cargo: 'CARGO' + cargo: 'CARGO', + terraform: 'TERRAFORM' ] def upperTables = [] allTables.each { tableName -> diff --git a/migration-nexus/src/test/java/com/github/klboke/kkrepo/migration/nexus/NexusApiMigrationServiceTest.java b/migration-nexus/src/test/java/com/github/klboke/kkrepo/migration/nexus/NexusApiMigrationServiceTest.java index 164d4696..0d00e814 100644 --- a/migration-nexus/src/test/java/com/github/klboke/kkrepo/migration/nexus/NexusApiMigrationServiceTest.java +++ b/migration-nexus/src/test/java/com/github/klboke/kkrepo/migration/nexus/NexusApiMigrationServiceTest.java @@ -2,11 +2,15 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.ObjectMapper; import com.github.klboke.kkrepo.core.RepositoryFormat; import com.github.klboke.kkrepo.core.RepositoryType; +import com.github.klboke.kkrepo.core.security.EncryptionSecrets; +import com.github.klboke.kkrepo.core.security.SecretCipher; +import com.github.klboke.kkrepo.core.security.TerraformSigningKeyMaterial; import com.github.klboke.kkrepo.migration.nexus.NexusApiMigrationService.NexusMigrationPreflight; import com.github.klboke.kkrepo.migration.nexus.NexusApiMigrationService.ConfigMigrationCounts; import com.github.klboke.kkrepo.migration.nexus.NexusApiMigrationService.NexusMigrationRequest; @@ -19,14 +23,35 @@ import com.github.klboke.kkrepo.migration.nexus.security.NexusSecurityExport; import com.github.klboke.kkrepo.persistence.jdbc.api.BlobStoreDao; import com.github.klboke.kkrepo.persistence.jdbc.api.RepositoryDao; +import com.github.klboke.kkrepo.persistence.jdbc.api.TerraformRegistryDao; import com.github.klboke.kkrepo.persistence.jdbc.api.model.BlobStoreRecord; import com.github.klboke.kkrepo.persistence.jdbc.api.model.RepositoryRecord; +import java.io.ByteArrayOutputStream; import java.lang.reflect.Proxy; +import java.nio.charset.StandardCharsets; +import java.security.KeyPairGenerator; +import java.security.Security; +import java.util.Date; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; +import org.bouncycastle.bcpg.ArmoredOutputStream; +import org.bouncycastle.bcpg.HashAlgorithmTags; +import org.bouncycastle.bcpg.SymmetricKeyAlgorithmTags; +import org.bouncycastle.bcpg.sig.KeyFlags; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.openpgp.PGPKeyPair; +import org.bouncycastle.openpgp.PGPKeyRingGenerator; +import org.bouncycastle.openpgp.PGPPublicKey; +import org.bouncycastle.openpgp.PGPSignature; +import org.bouncycastle.openpgp.PGPSignatureSubpacketGenerator; +import org.bouncycastle.openpgp.operator.PGPDigestCalculator; +import org.bouncycastle.openpgp.operator.jcajce.JcaPGPDigestCalculatorProviderBuilder; +import org.bouncycastle.openpgp.operator.jcajce.JcaPGPKeyPair; +import org.bouncycastle.openpgp.operator.jcajce.JcaPGPContentSignerBuilder; +import org.bouncycastle.openpgp.operator.jcajce.JcePBESecretKeyEncryptorBuilder; import org.junit.jupiter.api.Test; class NexusApiMigrationServiceTest { @@ -161,14 +186,15 @@ void datastoreSourceProfileEnablesRepositoryContentWhenSchemaFingerprintMatches( "DATASTORE_H2", "H2 2.3.232", "jdbc:h2:file:/nexus-data/db/nexus", - datastoreSchema(Map.of("maven2", true, "cargo", true)), + datastoreSchema(Map.of("maven2", true, "cargo", true, "terraform", true)), List.of()); NexusMigrationPreflight preflight = service.preflight(new NexusInventory( List.of(Map.of("name", "default", "type", "File")), List.of( repository("maven-releases", "maven2", "hosted", Map.of("storage", storage("default"))), - repository("cargo-hosted", "cargo", "hosted", Map.of("storage", storage("default")))), + repository("cargo-hosted", "cargo", "hosted", Map.of("storage", storage("default"))), + repository("terraform-hosted", "terraform", "hosted", Map.of("storage", storage("default")))), NexusSecurityExport.empty(), List.of(), probe), @@ -185,6 +211,9 @@ void datastoreSourceProfileEnablesRepositoryContentWhenSchemaFingerprintMatches( assertEquals( true, preflight.sourceProfile().formatCapabilities().get("cargo").contentMigration()); + assertEquals( + true, + preflight.sourceProfile().formatCapabilities().get("terraform").contentMigration()); assertTrue(preflight.warnings().stream() .noneMatch(value -> value.contains("Cargo migration remains configuration-only"))); assertEquals( @@ -201,6 +230,13 @@ void datastoreSourceProfileEnablesRepositoryContentWhenSchemaFingerprintMatches( .findFirst() .orElseThrow() .status()); + assertEquals( + SupportStatus.FULL, + preflight.migrationPlan().items().stream() + .filter(item -> "terraform-hosted".equals(item.name())) + .findFirst() + .orElseThrow() + .status()); } @Test @@ -503,6 +539,46 @@ void migratesDockerConnectorPortFromSourceHttpPort() { record.attributes().get("docker")); } + @Test + void importsTerraformSigningKeyAndRedactsSourceSecrets() throws Exception { + EncryptionSecrets.configure("terraform-migration-test-secret", null); + FakeBlobStoreDao blobStores = new FakeBlobStoreDao(); + FakeRepositoryDao repositories = new FakeRepositoryDao(); + FakeTerraformRegistryDao terraformRegistry = new FakeTerraformRegistryDao(); + NexusApiMigrationService service = new NexusApiMigrationService( + new ObjectMapper(), blobStores.asDao(), repositories.asDao(), null, null, null, + terraformRegistry.asDao()); + String passphrase = "source-key-passphrase"; + String privateKey = signingKey(passphrase); + NexusInventory inventory = new NexusInventory( + List.of(Map.of("name", "default")), + List.of(repository("terraform-hosted", "terraform", "hosted", Map.of( + "storage", storage("default"), + "terraformSigning", Map.of("keypair", privateKey, "passphrase", passphrase)))), + NexusSecurityExport.empty(), + List.of()); + + service.migrateConfig(inventory, request("https://old-nexus.example")); + RepositoryRecord repository = repositories.required("terraform-hosted"); + TerraformRegistryDao.SigningKey imported = terraformRegistry.key; + assertNotNull(imported); + assertEquals(repository.id().longValue(), imported.repositoryId()); + assertTrue(imported.publicKey().contains("BEGIN PGP PUBLIC KEY BLOCK")); + TerraformSigningKeyMaterial.Material keyMaterial = TerraformSigningKeyMaterial.decode( + new SecretCipher(EncryptionSecrets.credentialSecret()).decrypt(imported.encryptedPrivateKey())); + assertEquals(privateKey.strip(), keyMaterial.privateKeyArmor().strip()); + assertEquals(passphrase, keyMaterial.passphrase()); + @SuppressWarnings("unchecked") + Map source = (Map) repository.attributes().get("sourceRepository"); + @SuppressWarnings("unchecked") + Map signing = (Map) source.get("terraformSigning"); + assertEquals("", signing.get("keypair")); + assertEquals("", signing.get("passphrase")); + + service.migrateConfig(inventory, request("https://old-nexus.example")); + assertEquals(1, terraformRegistry.inserts); + } + private static NexusApiMigrationService service( FakeBlobStoreDao blobStores, FakeRepositoryDao repositories) { @@ -515,6 +591,35 @@ private static NexusApiMigrationService service( null); } + private static String signingKey(String passphrase) throws Exception { + if (Security.getProvider(BouncyCastleProvider.PROVIDER_NAME) == null) { + Security.addProvider(new BouncyCastleProvider()); + } + KeyPairGenerator generator = KeyPairGenerator.getInstance("RSA", BouncyCastleProvider.PROVIDER_NAME); + generator.initialize(2048); + PGPKeyPair pair = new JcaPGPKeyPair(PGPPublicKey.RSA_SIGN, generator.generateKeyPair(), new Date()); + PGPDigestCalculator sha1 = new JcaPGPDigestCalculatorProviderBuilder() + .setProvider(BouncyCastleProvider.PROVIDER_NAME).build().get(HashAlgorithmTags.SHA1); + PGPSignatureSubpacketGenerator certification = new PGPSignatureSubpacketGenerator(); + certification.setKeyFlags(false, KeyFlags.CERTIFY_OTHER | KeyFlags.SIGN_DATA); + PGPKeyRingGenerator rings = new PGPKeyRingGenerator( + PGPSignature.POSITIVE_CERTIFICATION, + pair, + "Terraform migration test ", + sha1, + certification.generate(), + null, + new JcaPGPContentSignerBuilder(pair.getPublicKey().getAlgorithm(), HashAlgorithmTags.SHA256) + .setProvider(BouncyCastleProvider.PROVIDER_NAME), + new JcePBESecretKeyEncryptorBuilder(SymmetricKeyAlgorithmTags.AES_256, sha1) + .setProvider(BouncyCastleProvider.PROVIDER_NAME).build(passphrase.toCharArray())); + ByteArrayOutputStream armored = new ByteArrayOutputStream(); + try (ArmoredOutputStream output = new ArmoredOutputStream(armored)) { + rings.generateSecretKeyRing().encode(output); + } + return armored.toString(StandardCharsets.UTF_8); + } + private static NexusMigrationRequest request(String sourceBaseUrl) { return new NexusMigrationRequest( sourceBaseUrl, @@ -686,4 +791,25 @@ private RepositoryRecord requiredById(Long id) { .orElseThrow(); } } + + private static final class FakeTerraformRegistryDao { + private TerraformRegistryDao.SigningKey key; + private int inserts; + + private TerraformRegistryDao asDao() { + return (TerraformRegistryDao) Proxy.newProxyInstance( + TerraformRegistryDao.class.getClassLoader(), + new Class[] {TerraformRegistryDao.class}, + (proxy, method, args) -> switch (method.getName()) { + case "findActiveSigningKey" -> key == null || key.repositoryId() != (Long) args[0] + ? Optional.empty() : Optional.of(key); + case "insertSigningKey" -> { + key = (TerraformRegistryDao.SigningKey) args[0]; + inserts++; + yield null; + } + default -> throw new UnsupportedOperationException(method.getName()); + }); + } + } } diff --git a/persistence-jdbc/src/main/java/com/github/klboke/kkrepo/persistence/jdbc/api/PersistenceStores.java b/persistence-jdbc/src/main/java/com/github/klboke/kkrepo/persistence/jdbc/api/PersistenceStores.java index 93f161f2..1886528c 100644 --- a/persistence-jdbc/src/main/java/com/github/klboke/kkrepo/persistence/jdbc/api/PersistenceStores.java +++ b/persistence-jdbc/src/main/java/com/github/klboke/kkrepo/persistence/jdbc/api/PersistenceStores.java @@ -42,6 +42,8 @@ public interface PersistenceStores extends AutoCloseable { SecurityDao security(); + TerraformRegistryDao terraformRegistry(); + UiSettingsDao uiSettings(); @Override diff --git a/persistence-jdbc/src/main/java/com/github/klboke/kkrepo/persistence/jdbc/api/TerraformRegistryDao.java b/persistence-jdbc/src/main/java/com/github/klboke/kkrepo/persistence/jdbc/api/TerraformRegistryDao.java new file mode 100644 index 00000000..282b6094 --- /dev/null +++ b/persistence-jdbc/src/main/java/com/github/klboke/kkrepo/persistence/jdbc/api/TerraformRegistryDao.java @@ -0,0 +1,73 @@ +package com.github.klboke.kkrepo.persistence.jdbc.api; + +import java.time.Instant; +import java.util.List; +import java.util.Optional; + +/** Shared state that makes Terraform provider publication and group routing replica-safe. */ +public interface TerraformRegistryDao { + Optional findActiveSigningKey(long repositoryId); + + Optional findSigningKey(long repositoryId, int revision); + + void insertSigningKey(SigningKey key); + + Optional findProviderState( + long repositoryId, String namespace, String type, String version); + + List listProviderPlatforms( + long repositoryId, String namespace, String type, String version); + + void publishProvider(ProviderPlatform platform, ProviderState state); + + boolean tryAcquirePublishLease(String leaseKey, String owner, Instant expiresAt); + + void releasePublishLease(String leaseKey, String owner); + + Optional findSourceBinding(long groupRepositoryId, String bindingKey); + + void upsertSourceBinding(SourceBinding binding); + + void deleteSourceBindings(long groupRepositoryId); + + record SigningKey( + long repositoryId, + int revision, + String keyId, + String encryptedPrivateKey, + String publicKey, + Instant createdAt) {} + + record ProviderState( + long repositoryId, + String namespace, + String type, + String version, + long revision, + String shasumsPath, + String signaturePath, + int signingKeyRevision, + Instant updatedAt) {} + + record ProviderPlatform( + long repositoryId, + String namespace, + String type, + String version, + String os, + String arch, + String filename, + String assetPath, + String sha256, + String protocols, + long revision, + Instant updatedAt) {} + + record SourceBinding( + long groupRepositoryId, + String bindingKey, + long memberRepositoryId, + long memberRevision, + Instant expiresAt, + Instant updatedAt) {} +} diff --git a/persistence-jdbc/src/main/java/com/github/klboke/kkrepo/persistence/jdbc/internal/JdbcPersistenceStoreFactory.java b/persistence-jdbc/src/main/java/com/github/klboke/kkrepo/persistence/jdbc/internal/JdbcPersistenceStoreFactory.java index dff3e567..10509e1c 100644 --- a/persistence-jdbc/src/main/java/com/github/klboke/kkrepo/persistence/jdbc/internal/JdbcPersistenceStoreFactory.java +++ b/persistence-jdbc/src/main/java/com/github/klboke/kkrepo/persistence/jdbc/internal/JdbcPersistenceStoreFactory.java @@ -24,6 +24,7 @@ import com.github.klboke.kkrepo.persistence.jdbc.api.RepositoryIndexRebuildDao; import com.github.klboke.kkrepo.persistence.jdbc.api.SecurityAuditDao; import com.github.klboke.kkrepo.persistence.jdbc.api.SecurityDao; +import com.github.klboke.kkrepo.persistence.jdbc.api.TerraformRegistryDao; import com.github.klboke.kkrepo.persistence.jdbc.api.UiSettingsDao; import com.github.klboke.kkrepo.persistence.jdbc.internal.support.JsonColumns; import com.github.klboke.kkrepo.persistence.jdbc.spi.DatabaseDialect; @@ -68,6 +69,7 @@ public static PersistenceStores createStores(JdbcTemplate jdbc, DatabaseDialect new JdbcRepositoryIndexRebuildDao(jdbc, dialect), new JdbcSecurityAuditDao(jdbc, json), new JdbcSecurityDao(jdbc, json, dialect), + new JdbcTerraformRegistryDao(jdbc), new JdbcUiSettingsDao(jdbc)); } @@ -92,6 +94,7 @@ private record DefaultPersistenceStores( RepositoryIndexRebuildDao repositoryIndexRebuild, SecurityAuditDao securityAudit, SecurityDao security, + TerraformRegistryDao terraformRegistry, UiSettingsDao uiSettings) implements PersistenceStores { } } diff --git a/persistence-jdbc/src/main/java/com/github/klboke/kkrepo/persistence/jdbc/internal/JdbcTerraformRegistryDao.java b/persistence-jdbc/src/main/java/com/github/klboke/kkrepo/persistence/jdbc/internal/JdbcTerraformRegistryDao.java new file mode 100644 index 00000000..a69d0595 --- /dev/null +++ b/persistence-jdbc/src/main/java/com/github/klboke/kkrepo/persistence/jdbc/internal/JdbcTerraformRegistryDao.java @@ -0,0 +1,190 @@ +package com.github.klboke.kkrepo.persistence.jdbc.internal; + +import static com.github.klboke.kkrepo.persistence.jdbc.internal.support.JdbcRows.nullableInstant; +import static com.github.klboke.kkrepo.persistence.jdbc.internal.support.JdbcRows.nullableTimestamp; + +import com.github.klboke.kkrepo.persistence.jdbc.api.TerraformRegistryDao; +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; + +@Repository +public class JdbcTerraformRegistryDao implements TerraformRegistryDao { + private final JdbcTemplate jdbc; + + public JdbcTerraformRegistryDao(JdbcTemplate jdbc) { + this.jdbc = jdbc; + } + + @Override + public Optional findActiveSigningKey(long repositoryId) { + return jdbc.query(""" + SELECT * FROM terraform_signing_key + WHERE repository_id = ? AND active = TRUE + ORDER BY revision DESC + """, (rs, row) -> new SigningKey( + rs.getLong("repository_id"), rs.getInt("revision"), rs.getString("key_id"), + rs.getString("encrypted_private_key"), rs.getString("public_key"), + nullableInstant(rs, "created_at")), repositoryId).stream().findFirst(); + } + + @Override + public Optional findSigningKey(long repositoryId, int revision) { + return jdbc.query(""" + SELECT * FROM terraform_signing_key WHERE repository_id = ? AND revision = ? + """, (rs, row) -> new SigningKey( + rs.getLong("repository_id"), rs.getInt("revision"), rs.getString("key_id"), + rs.getString("encrypted_private_key"), rs.getString("public_key"), + nullableInstant(rs, "created_at")), repositoryId, revision).stream().findFirst(); + } + + @Override + @org.springframework.transaction.annotation.Transactional + public void insertSigningKey(SigningKey key) { + jdbc.update("UPDATE terraform_signing_key SET active = FALSE WHERE repository_id = ?", key.repositoryId()); + jdbc.update(""" + INSERT INTO terraform_signing_key + (repository_id, revision, key_id, encrypted_private_key, public_key, active, created_at) + VALUES (?, ?, ?, ?, ?, TRUE, ?) + """, key.repositoryId(), key.revision(), key.keyId(), key.encryptedPrivateKey(), + key.publicKey(), nullableTimestamp(key.createdAt())); + } + + @Override + public Optional findProviderState( + long repositoryId, String namespace, String type, String version) { + return jdbc.query(""" + SELECT * FROM terraform_provider_signing_state + WHERE repository_id = ? AND namespace = ? AND provider_type = ? AND version = ? + """, (rs, row) -> new ProviderState( + rs.getLong("repository_id"), rs.getString("namespace"), rs.getString("provider_type"), + rs.getString("version"), rs.getLong("revision"), rs.getString("shasums_path"), + rs.getString("signature_path"), rs.getInt("signing_key_revision"), + nullableInstant(rs, "updated_at")), repositoryId, namespace, type, version) + .stream().findFirst(); + } + + @Override + public List listProviderPlatforms( + long repositoryId, String namespace, String type, String version) { + return jdbc.query(""" + SELECT * FROM terraform_provider_platform + WHERE repository_id = ? AND namespace = ? AND provider_type = ? AND version = ? + AND status = 'READY' + ORDER BY os, arch + """, (rs, row) -> new ProviderPlatform( + rs.getLong("repository_id"), rs.getString("namespace"), rs.getString("provider_type"), + rs.getString("version"), rs.getString("os"), rs.getString("arch"), + rs.getString("filename"), rs.getString("asset_path"), rs.getString("sha256"), + rs.getString("protocols"), rs.getLong("revision"), nullableInstant(rs, "updated_at")), + repositoryId, namespace, type, version); + } + + @Override + @org.springframework.transaction.annotation.Transactional + public void publishProvider(ProviderPlatform platform, ProviderState state) { + int updated = jdbc.update(""" + UPDATE terraform_provider_platform + SET filename = ?, asset_path = ?, sha256 = ?, protocols = ?, status = 'READY', + revision = ?, updated_at = ? + WHERE repository_id = ? AND namespace = ? AND provider_type = ? AND version = ? + AND os = ? AND arch = ? + """, platform.filename(), platform.assetPath(), platform.sha256(), platform.protocols(), + platform.revision(), nullableTimestamp(platform.updatedAt()), platform.repositoryId(), + platform.namespace(), platform.type(), platform.version(), platform.os(), platform.arch()); + if (updated == 0) { + jdbc.update(""" + INSERT INTO terraform_provider_platform + (repository_id, namespace, provider_type, version, os, arch, filename, asset_path, + sha256, protocols, status, revision, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'READY', ?, ?) + """, platform.repositoryId(), platform.namespace(), platform.type(), platform.version(), + platform.os(), platform.arch(), platform.filename(), platform.assetPath(), platform.sha256(), + platform.protocols(), platform.revision(), nullableTimestamp(platform.updatedAt())); + } + updated = jdbc.update(""" + UPDATE terraform_provider_signing_state + SET revision = ?, shasums_path = ?, signature_path = ?, signing_key_revision = ?, updated_at = ? + WHERE repository_id = ? AND namespace = ? AND provider_type = ? AND version = ? + """, state.revision(), state.shasumsPath(), state.signaturePath(), state.signingKeyRevision(), + nullableTimestamp(state.updatedAt()), state.repositoryId(), state.namespace(), state.type(), state.version()); + if (updated == 0) { + jdbc.update(""" + INSERT INTO terraform_provider_signing_state + (repository_id, namespace, provider_type, version, revision, shasums_path, + signature_path, signing_key_revision, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, state.repositoryId(), state.namespace(), state.type(), state.version(), state.revision(), + state.shasumsPath(), state.signaturePath(), state.signingKeyRevision(), + nullableTimestamp(state.updatedAt())); + } + } + + @Override + public boolean tryAcquirePublishLease(String leaseKey, String owner, Instant expiresAt) { + Instant now = Instant.now(); + int updated = jdbc.update(""" + UPDATE terraform_publish_lease + SET owner = ?, expires_at = ?, updated_at = ? + WHERE lease_key = ? AND (expires_at < ? OR owner = ?) + """, owner, nullableTimestamp(expiresAt), nullableTimestamp(now), leaseKey, + nullableTimestamp(now), owner); + if (updated > 0) return true; + try { + jdbc.update(""" + INSERT INTO terraform_publish_lease (lease_key, owner, expires_at, updated_at) + VALUES (?, ?, ?, ?) + """, leaseKey, owner, nullableTimestamp(expiresAt), nullableTimestamp(now)); + return true; + } catch (DuplicateKeyException e) { + return false; + } + } + + @Override + public void releasePublishLease(String leaseKey, String owner) { + jdbc.update("DELETE FROM terraform_publish_lease WHERE lease_key = ? AND owner = ?", leaseKey, owner); + } + + @Override + public Optional findSourceBinding(long groupRepositoryId, String bindingKey) { + return jdbc.query(""" + SELECT * FROM terraform_source_binding + WHERE group_repository_id = ? AND binding_key = ? AND expires_at > ? + """, (rs, row) -> new SourceBinding( + rs.getLong("group_repository_id"), rs.getString("binding_key"), + rs.getLong("member_repository_id"), rs.getLong("member_revision"), + nullableInstant(rs, "expires_at"), nullableInstant(rs, "updated_at")), + groupRepositoryId, bindingKey, nullableTimestamp(Instant.now())).stream().findFirst(); + } + + @Override + public void upsertSourceBinding(SourceBinding binding) { + int updated = jdbc.update(""" + UPDATE terraform_source_binding + SET member_repository_id = ?, member_revision = ?, expires_at = ?, updated_at = ? + WHERE group_repository_id = ? AND binding_key = ? + """, binding.memberRepositoryId(), binding.memberRevision(), nullableTimestamp(binding.expiresAt()), + nullableTimestamp(binding.updatedAt()), binding.groupRepositoryId(), binding.bindingKey()); + if (updated == 0) { + try { + jdbc.update(""" + INSERT INTO terraform_source_binding + (group_repository_id, binding_key, member_repository_id, member_revision, expires_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + """, binding.groupRepositoryId(), binding.bindingKey(), binding.memberRepositoryId(), + binding.memberRevision(), nullableTimestamp(binding.expiresAt()), nullableTimestamp(binding.updatedAt())); + } catch (DuplicateKeyException e) { + upsertSourceBinding(binding); + } + } + } + + @Override + public void deleteSourceBindings(long groupRepositoryId) { + jdbc.update("DELETE FROM terraform_source_binding WHERE group_repository_id = ?", groupRepositoryId); + } +} diff --git a/persistence-jdbc/src/test/java/com/github/klboke/kkrepo/persistence/jdbc/contract/PersistenceApiContract.java b/persistence-jdbc/src/test/java/com/github/klboke/kkrepo/persistence/jdbc/contract/PersistenceApiContract.java index 7597a9da..91fc6dfb 100644 --- a/persistence-jdbc/src/test/java/com/github/klboke/kkrepo/persistence/jdbc/contract/PersistenceApiContract.java +++ b/persistence-jdbc/src/test/java/com/github/klboke/kkrepo/persistence/jdbc/contract/PersistenceApiContract.java @@ -14,6 +14,7 @@ import com.github.klboke.kkrepo.persistence.jdbc.api.RepositoryDataMigrationDao; import com.github.klboke.kkrepo.persistence.jdbc.api.RepositoryIndexRebuildDao; import com.github.klboke.kkrepo.persistence.jdbc.api.SecurityAuditDao; +import com.github.klboke.kkrepo.persistence.jdbc.api.TerraformRegistryDao; import com.github.klboke.kkrepo.persistence.jdbc.api.model.AssetBlobRecord; import com.github.klboke.kkrepo.persistence.jdbc.api.model.AssetRecord; import com.github.klboke.kkrepo.persistence.jdbc.api.model.BlobStoreRecord; @@ -98,9 +99,57 @@ void baselineContainsTheCompleteSharedLogicalSchema() { "security_user_role", "spring_session", "spring_session_attributes", + "terraform_provider_platform", + "terraform_provider_signing_state", + "terraform_publish_lease", + "terraform_signing_key", + "terraform_source_binding", "ui_settings"), databaseTables()); } + @Test + void terraformRegistryStateIsSharedTransactionalAndReplicaSafe() { + long repositoryId = createRepository("terraform-hosted", RepositoryFormat.TERRAFORM); + long memberRepositoryId = createRepository("terraform-member", RepositoryFormat.TERRAFORM); + TerraformRegistryDao registry = stores().terraformRegistry(); + Instant now = Instant.now(); + registry.insertSigningKey(new TerraformRegistryDao.SigningKey( + repositoryId, 1, "0123456789ABCDEF", "encrypted-private", "public-key", now)); + + assertEquals("0123456789ABCDEF", registry.findActiveSigningKey(repositoryId).orElseThrow().keyId()); + assertEquals(1, registry.findSigningKey(repositoryId, 1).orElseThrow().revision()); + + registry.publishProvider( + new TerraformRegistryDao.ProviderPlatform( + repositoryId, "kkrepo", "fixture", "1.2.3", "linux", "amd64", + "terraform-provider-fixture_1.2.3_linux_amd64.zip", + "v1/providers/kkrepo/fixture/1.2.3/package/linux/terraform-provider-fixture-fixture.zip", + "a".repeat(64), "5.0", 1, now), + new TerraformRegistryDao.ProviderState( + repositoryId, "kkrepo", "fixture", "1.2.3", 1, + "v1/providers/kkrepo/fixture/1.2.3/terraform-provider-fixture_1.2.3_SHA256SUMS", + "v1/providers/kkrepo/fixture/1.2.3/terraform-provider-fixture_1.2.3_SHA256SUMS.sig", + 1, now)); + assertEquals(1, registry.listProviderPlatforms( + repositoryId, "kkrepo", "fixture", "1.2.3").size()); + assertEquals(1, registry.findProviderState( + repositoryId, "kkrepo", "fixture", "1.2.3").orElseThrow().revision()); + + assertTrue(registry.tryAcquirePublishLease("provider:fixture", "replica-a", now.plusSeconds(30))); + assertFalse(registry.tryAcquirePublishLease("provider:fixture", "replica-b", now.plusSeconds(30))); + registry.releasePublishLease("provider:fixture", "replica-a"); + assertTrue(registry.tryAcquirePublishLease("provider:fixture", "replica-b", now.plusSeconds(30))); + + registry.upsertSourceBinding(new TerraformRegistryDao.SourceBinding( + repositoryId, "asset:v1/providers/kkrepo/fixture", memberRepositoryId, 7, + now.plusSeconds(60), now)); + assertEquals(memberRepositoryId, registry.findSourceBinding( + repositoryId, "asset:v1/providers/kkrepo/fixture").orElseThrow().memberRepositoryId()); + registry.deleteSourceBindings(repositoryId); + assertTrue(registry.findSourceBinding( + repositoryId, "asset:v1/providers/kkrepo/fixture").isEmpty()); + } + @Test void componentUpsertIsAtomicAndJsonValuesRoundTrip() throws Exception { long repositoryId = createRepository("maven-hosted", RepositoryFormat.MAVEN2); diff --git a/persistence-mysql/src/main/resources/db/migration/mysql/V30__terraform_registry.sql b/persistence-mysql/src/main/resources/db/migration/mysql/V30__terraform_registry.sql new file mode 100644 index 00000000..f0f29945 --- /dev/null +++ b/persistence-mysql/src/main/resources/db/migration/mysql/V30__terraform_registry.sql @@ -0,0 +1,67 @@ +CREATE TABLE terraform_signing_key ( + id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY, + repository_id BIGINT UNSIGNED NOT NULL, + revision INT NOT NULL, + key_id VARCHAR(32) NOT NULL, + encrypted_private_key LONGTEXT NOT NULL, + public_key LONGTEXT NOT NULL, + active BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + CONSTRAINT fk_terraform_signing_repository FOREIGN KEY (repository_id) REFERENCES repository(id) ON DELETE CASCADE, + CONSTRAINT uk_terraform_signing_revision UNIQUE (repository_id, revision), + INDEX idx_terraform_signing_active (repository_id, active) +); + +CREATE TABLE terraform_provider_signing_state ( + repository_id BIGINT UNSIGNED NOT NULL, + namespace VARCHAR(128) NOT NULL, + provider_type VARCHAR(128) NOT NULL, + version VARCHAR(128) NOT NULL, + revision BIGINT NOT NULL, + shasums_path VARCHAR(1024) NOT NULL, + signature_path VARCHAR(1024) NOT NULL, + signing_key_revision INT NOT NULL, + updated_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + PRIMARY KEY (repository_id, namespace, provider_type, version), + CONSTRAINT fk_terraform_state_repository FOREIGN KEY (repository_id) REFERENCES repository(id) ON DELETE CASCADE +); + +CREATE TABLE terraform_provider_platform ( + repository_id BIGINT UNSIGNED NOT NULL, + namespace VARCHAR(128) NOT NULL, + provider_type VARCHAR(128) NOT NULL, + version VARCHAR(128) NOT NULL, + os VARCHAR(64) NOT NULL, + arch VARCHAR(64) NOT NULL, + filename VARCHAR(255) NOT NULL, + asset_path VARCHAR(1024) NOT NULL, + sha256 CHAR(64) NOT NULL, + protocols VARCHAR(255) NOT NULL, + status VARCHAR(16) NOT NULL, + revision BIGINT NOT NULL, + updated_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + PRIMARY KEY (repository_id, namespace, provider_type, version, os, arch), + CONSTRAINT fk_terraform_platform_repository FOREIGN KEY (repository_id) REFERENCES repository(id) ON DELETE CASCADE, + INDEX idx_terraform_platform_versions (repository_id, namespace, provider_type, version, status) +); + +CREATE TABLE terraform_source_binding ( + group_repository_id BIGINT UNSIGNED NOT NULL, + binding_key VARCHAR(512) NOT NULL, + member_repository_id BIGINT UNSIGNED NOT NULL, + member_revision BIGINT NOT NULL, + expires_at TIMESTAMP(6) NOT NULL, + updated_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + PRIMARY KEY (group_repository_id, binding_key), + CONSTRAINT fk_terraform_binding_group FOREIGN KEY (group_repository_id) REFERENCES repository(id) ON DELETE CASCADE, + CONSTRAINT fk_terraform_binding_member FOREIGN KEY (member_repository_id) REFERENCES repository(id) ON DELETE CASCADE, + INDEX idx_terraform_binding_expiry (expires_at) +); + +CREATE TABLE terraform_publish_lease ( + lease_key VARCHAR(512) NOT NULL PRIMARY KEY, + owner VARCHAR(128) NOT NULL, + expires_at TIMESTAMP(6) NOT NULL, + updated_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + INDEX idx_terraform_lease_expiry (expires_at) +); diff --git a/persistence-mysql/src/test/java/com/github/klboke/kkrepo/persistence/mysql/MySqlV29MigrationCompatibilityTest.java b/persistence-mysql/src/test/java/com/github/klboke/kkrepo/persistence/mysql/MySqlV29MigrationCompatibilityTest.java index c8f417e6..60d14020 100644 --- a/persistence-mysql/src/test/java/com/github/klboke/kkrepo/persistence/mysql/MySqlV29MigrationCompatibilityTest.java +++ b/persistence-mysql/src/test/java/com/github/klboke/kkrepo/persistence/mysql/MySqlV29MigrationCompatibilityTest.java @@ -10,7 +10,7 @@ import java.util.Map; import org.junit.jupiter.api.Test; -/** Guards the relocated legacy migration chain and validates repeat startup at V29. */ +/** Guards the frozen V1-V29 chain and validates repeat startup after newer migrations. */ class MySqlV29MigrationCompatibilityTest extends MySqlIntegrationTestSupport { private static final Map V29_SHA256 = checksums(); @@ -28,11 +28,11 @@ void relocatedV1ThroughV29RemainByteForByteFrozen() throws Exception { } @Test - void existingV29HistoryValidatesAndSecondMigrateHasNoPendingWork() { + void existingV29HistoryUpgradesAndSecondMigrateHasNoPendingWork() { assertTrue(flyway().validateWithResult().validationSuccessful); var result = flyway().migrate(); assertEquals(0, result.migrationsExecuted); - assertEquals("29", flyway().info().current().getVersion().getVersion()); + assertEquals("30", flyway().info().current().getVersion().getVersion()); } private static String hex(byte[] bytes) { diff --git a/persistence-postgresql/src/main/resources/db/migration/postgresql/V30__terraform_registry.sql b/persistence-postgresql/src/main/resources/db/migration/postgresql/V30__terraform_registry.sql new file mode 100644 index 00000000..b44dabcc --- /dev/null +++ b/persistence-postgresql/src/main/resources/db/migration/postgresql/V30__terraform_registry.sql @@ -0,0 +1,63 @@ +CREATE TABLE terraform_signing_key ( + id BIGSERIAL PRIMARY KEY, + repository_id BIGINT NOT NULL REFERENCES repository(id) ON DELETE CASCADE, + revision INTEGER NOT NULL, + key_id VARCHAR(32) NOT NULL, + encrypted_private_key TEXT NOT NULL, + public_key TEXT NOT NULL, + active BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT uk_terraform_signing_revision UNIQUE (repository_id, revision) +); +CREATE INDEX idx_terraform_signing_active ON terraform_signing_key(repository_id, active); + +CREATE TABLE terraform_provider_signing_state ( + repository_id BIGINT NOT NULL REFERENCES repository(id) ON DELETE CASCADE, + namespace VARCHAR(128) NOT NULL, + provider_type VARCHAR(128) NOT NULL, + version VARCHAR(128) NOT NULL, + revision BIGINT NOT NULL, + shasums_path VARCHAR(1024) NOT NULL, + signature_path VARCHAR(1024) NOT NULL, + signing_key_revision INTEGER NOT NULL, + updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (repository_id, namespace, provider_type, version) +); + +CREATE TABLE terraform_provider_platform ( + repository_id BIGINT NOT NULL REFERENCES repository(id) ON DELETE CASCADE, + namespace VARCHAR(128) NOT NULL, + provider_type VARCHAR(128) NOT NULL, + version VARCHAR(128) NOT NULL, + os VARCHAR(64) NOT NULL, + arch VARCHAR(64) NOT NULL, + filename VARCHAR(255) NOT NULL, + asset_path VARCHAR(1024) NOT NULL, + sha256 CHAR(64) NOT NULL, + protocols VARCHAR(255) NOT NULL, + status VARCHAR(16) NOT NULL, + revision BIGINT NOT NULL, + updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (repository_id, namespace, provider_type, version, os, arch) +); +CREATE INDEX idx_terraform_platform_versions + ON terraform_provider_platform(repository_id, namespace, provider_type, version, status); + +CREATE TABLE terraform_source_binding ( + group_repository_id BIGINT NOT NULL REFERENCES repository(id) ON DELETE CASCADE, + binding_key VARCHAR(512) NOT NULL, + member_repository_id BIGINT NOT NULL REFERENCES repository(id) ON DELETE CASCADE, + member_revision BIGINT NOT NULL, + expires_at TIMESTAMP WITHOUT TIME ZONE NOT NULL, + updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (group_repository_id, binding_key) +); +CREATE INDEX idx_terraform_binding_expiry ON terraform_source_binding(expires_at); + +CREATE TABLE terraform_publish_lease ( + lease_key VARCHAR(512) PRIMARY KEY, + owner VARCHAR(128) NOT NULL, + expires_at TIMESTAMP WITHOUT TIME ZONE NOT NULL, + updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE INDEX idx_terraform_lease_expiry ON terraform_publish_lease(expires_at); diff --git a/persistence-postgresql/src/test/java/com/github/klboke/kkrepo/persistence/postgresql/PostgreSqlMigrationCompatibilityTest.java b/persistence-postgresql/src/test/java/com/github/klboke/kkrepo/persistence/postgresql/PostgreSqlMigrationCompatibilityTest.java index 013c62c6..60aab5d9 100644 --- a/persistence-postgresql/src/test/java/com/github/klboke/kkrepo/persistence/postgresql/PostgreSqlMigrationCompatibilityTest.java +++ b/persistence-postgresql/src/test/java/com/github/klboke/kkrepo/persistence/postgresql/PostgreSqlMigrationCompatibilityTest.java @@ -13,6 +13,6 @@ void baselineValidatesAndSecondMigrateHasNoPendingWork() { assertTrue(flyway().validateWithResult().validationSuccessful); var result = flyway().migrate(); assertEquals(0, result.migrationsExecuted); - assertEquals("29", flyway().info().current().getVersion().getVersion()); + assertEquals("30", flyway().info().current().getVersion().getVersion()); } } diff --git a/pom.xml b/pom.xml index 7a72eaf0..04860a41 100644 --- a/pom.xml +++ b/pom.xml @@ -23,6 +23,7 @@ protocol-cargo protocol-pub protocol-composer + protocol-terraform protocol-go protocol-helm protocol-docker @@ -50,7 +51,9 @@ 3.9.16 2.47.5 1.28.0 + 1.10 3.20.0 + 1.83 0.8.15 1.7.3 ${java.version} @@ -112,11 +115,26 @@ commons-compress ${commons.compress.version} + + org.tukaani + xz + ${xz.version} + org.apache.commons commons-lang3 ${commons.lang3.version} + + org.bouncycastle + bcprov-jdk18on + ${bouncycastle.version} + + + org.bouncycastle + bcpg-jdk18on + ${bouncycastle.version} + software.amazon.awssdk bom @@ -174,6 +192,11 @@ kkrepo-protocol-composer ${project.version} + + com.github.klboke + kkrepo-protocol-terraform + ${project.version} + com.github.klboke kkrepo-protocol-go diff --git a/protocol-terraform/pom.xml b/protocol-terraform/pom.xml new file mode 100644 index 00000000..4b10f239 --- /dev/null +++ b/protocol-terraform/pom.xml @@ -0,0 +1,25 @@ + + + 4.0.0 + + com.github.klboke + kkrepo + ${revision} + ../pom.xml + + kkrepo-protocol-terraform + kkrepo-protocol-terraform + + + com.github.klboke + kkrepo-core + + + org.junit.jupiter + junit-jupiter + test + + + diff --git a/protocol-terraform/src/main/java/com/github/klboke/kkrepo/protocol/terraform/TerraformPath.java b/protocol-terraform/src/main/java/com/github/klboke/kkrepo/protocol/terraform/TerraformPath.java new file mode 100644 index 00000000..4202b2b4 --- /dev/null +++ b/protocol-terraform/src/main/java/com/github/klboke/kkrepo/protocol/terraform/TerraformPath.java @@ -0,0 +1,39 @@ +package com.github.klboke.kkrepo.protocol.terraform; + +/** A validated Terraform Registry Protocol route relative to a repository root. */ +public record TerraformPath( + Kind kind, + String namespace, + String name, + String system, + String version, + String os, + String arch, + String filename, + String credentialSegment, + String rawPath) { + + public enum Kind { + MODULE_VERSIONS, + MODULE_DOWNLOAD, + MODULE_ARCHIVE, + PROVIDER_VERSIONS, + PROVIDER_DOWNLOAD, + PROVIDER_ARCHIVE, + PROVIDER_SHA256SUMS, + PROVIDER_SHA256SUMS_SIGNATURE, + UNKNOWN + } + + public boolean module() { + return kind == Kind.MODULE_VERSIONS || kind == Kind.MODULE_DOWNLOAD || kind == Kind.MODULE_ARCHIVE; + } + + public boolean provider() { + return !module() && kind != Kind.UNKNOWN; + } + + public String coordinate() { + return namespace + "/" + name; + } +} diff --git a/protocol-terraform/src/main/java/com/github/klboke/kkrepo/protocol/terraform/TerraformPathParser.java b/protocol-terraform/src/main/java/com/github/klboke/kkrepo/protocol/terraform/TerraformPathParser.java new file mode 100644 index 00000000..0e03af83 --- /dev/null +++ b/protocol-terraform/src/main/java/com/github/klboke/kkrepo/protocol/terraform/TerraformPathParser.java @@ -0,0 +1,189 @@ +package com.github.klboke.kkrepo.protocol.terraform; + +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.regex.Pattern; + +/** Strict parser for the HashiCorp/Nexus Terraform registry paths. */ +public final class TerraformPathParser { + private static final Pattern ID = Pattern.compile("^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$"); + private static final Pattern VERSION = Pattern.compile( + "^(?:0|[1-9][0-9]*)(?:\\.(?:0|[1-9][0-9]*)){2}(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$"); + + public TerraformPath parse(String rawPath) { + String raw = normalize(rawPath); + List segments = splitAndValidate(raw); + if (segments.size() < 2 || !"v1".equals(segments.get(0))) return unknown(raw); + return switch (segments.get(1)) { + case "modules" -> parseModules(raw, segments); + case "providers" -> parseProviders(raw, segments); + default -> unknown(raw); + }; + } + + /** + * Parses an optional Nexus URL-token segment and returns the canonical path consumed by protocol + * services. Direct protocol routes always win, so a namespace is never guessed to be a token. + */ + public ParsedRequest parseRequestPath(String rawPath) { + String raw = normalize(rawPath); + TerraformPath direct; + IllegalArgumentException directFailure = null; + try { + direct = parse(raw); + } catch (IllegalArgumentException e) { + direct = unknown(raw); + directFailure = e; + } + if (direct.kind() != TerraformPath.Kind.UNKNOWN) return new ParsedRequest(raw, null, direct); + List segments = splitAndValidate(raw); + if (segments.size() <= 2 || !"v1".equals(segments.get(0)) + || !("modules".equals(segments.get(1)) || "providers".equals(segments.get(1)))) { + return new ParsedRequest(raw, null, direct); + } + String credential = decodeOnce(segments.get(2)); + if (credential.length() > 4096) throw new IllegalArgumentException("Terraform URL token is too long"); + List canonical = new ArrayList<>(segments); + canonical.remove(2); + String normalized = String.join("/", canonical); + TerraformPath parsed; + try { + parsed = parse(normalized); + } catch (IllegalArgumentException e) { + if (directFailure != null) throw directFailure; + throw e; + } + if (parsed.kind() == TerraformPath.Kind.UNKNOWN) { + if (directFailure != null) throw directFailure; + return new ParsedRequest(raw, null, direct); + } + return new ParsedRequest(normalized, credential, parsed); + } + + private TerraformPath parseModules(String raw, List input) { + List s = input; + if (s.size() == 6 && "versions".equals(s.get(5))) { + return module(TerraformPath.Kind.MODULE_VERSIONS, s, null, null, null, raw); + } + if (s.size() == 7 && "download".equals(s.get(6))) { + requireVersion(s.get(5)); + return module(TerraformPath.Kind.MODULE_DOWNLOAD, s, null, s.get(5), null, raw); + } + if (s.size() == 7) { + requireVersion(s.get(5)); + requireFilename(s.get(6)); + return module(TerraformPath.Kind.MODULE_ARCHIVE, s, null, s.get(5), s.get(6), raw); + } + return unknown(raw); + } + + private TerraformPath parseProviders(String raw, List input) { + List s = input; + if (s.size() == 5 && "versions".equals(s.get(4))) { + return provider(TerraformPath.Kind.PROVIDER_VERSIONS, s, null, null, null, null, raw); + } + if (s.size() == 8 && "download".equals(s.get(5))) { + requireVersion(s.get(4)); + return provider(TerraformPath.Kind.PROVIDER_DOWNLOAD, s, null, s.get(4), s.get(6), s.get(7), raw); + } + if (s.size() == 8 && "package".equals(s.get(5))) { + requireVersion(s.get(4)); + requireFilename(s.get(7)); + return provider(TerraformPath.Kind.PROVIDER_ARCHIVE, s, null, s.get(4), s.get(6), null, s.get(7), raw); + } + if (s.size() == 7) { + requireVersion(s.get(4)); + String filename = s.get(6); + requireFilename(filename); + TerraformPath.Kind kind = filename.endsWith("_SHA256SUMS.sig") + ? TerraformPath.Kind.PROVIDER_SHA256SUMS_SIGNATURE + : filename.endsWith("_SHA256SUMS") + ? TerraformPath.Kind.PROVIDER_SHA256SUMS + : TerraformPath.Kind.UNKNOWN; + return provider(kind, s, null, s.get(4), null, null, filename, raw); + } + return unknown(raw); + } + + private TerraformPath module(TerraformPath.Kind kind, List s, String credential, + String version, String filename, String raw) { + requireId(s.get(2), "namespace"); + requireId(s.get(3), "module name"); + requireId(s.get(4), "system"); + return new TerraformPath(kind, s.get(2), s.get(3), s.get(4), version, + null, null, filename, credential, raw); + } + + private TerraformPath provider(TerraformPath.Kind kind, List s, String credential, + String version, String os, String arch, String raw) { + return provider(kind, s, credential, version, os, arch, null, raw); + } + + private TerraformPath provider(TerraformPath.Kind kind, List s, String credential, + String version, String os, String arch, String filename, String raw) { + requireId(s.get(2), "namespace"); + requireId(s.get(3), "provider type"); + if (os != null) requireId(os, "os"); + if (arch != null) requireId(arch, "architecture"); + return new TerraformPath(kind, s.get(2), s.get(3), null, version, + os, arch, filename, credential, raw); + } + + private static List splitAndValidate(String raw) { + if (raw.indexOf('\0') >= 0 || raw.toLowerCase(Locale.ROOT).contains("%00") + || raw.toLowerCase(Locale.ROOT).contains("%2f") || raw.toLowerCase(Locale.ROOT).contains("%5c") + || raw.toLowerCase(Locale.ROOT).contains("%25")) { + throw new IllegalArgumentException("Unsafe Terraform repository path"); + } + String[] values = raw.split("/", -1); + List result = new ArrayList<>(values.length); + for (String value : values) { + if (value.isEmpty() || ".".equals(value) || "..".equals(value) || value.indexOf('\\') >= 0) { + throw new IllegalArgumentException("Unsafe Terraform repository path segment"); + } + result.add(value); + } + return List.copyOf(result); + } + + private static String normalize(String rawPath) { + String value = rawPath == null ? "" : rawPath.trim(); + while (value.startsWith("/")) value = value.substring(1); + while (value.endsWith("/")) value = value.substring(0, value.length() - 1); + return value; + } + + private static void requireId(String value, String label) { + if (!ID.matcher(value).matches()) throw new IllegalArgumentException("Invalid Terraform " + label); + } + + private static void requireVersion(String value) { + if (!VERSION.matcher(value).matches()) throw new IllegalArgumentException("Invalid Terraform semantic version"); + } + + public static void requireFilename(String value) { + if (value == null || value.isBlank() || value.length() > 255 || value.contains("/") + || value.contains("\\") || value.contains("\r") || value.contains("\n") + || ".".equals(value) || "..".equals(value)) { + throw new IllegalArgumentException("Invalid Terraform archive filename"); + } + } + + private static String decodeOnce(String value) { + try { + return URLDecoder.decode(value, StandardCharsets.UTF_8); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid Terraform URL token encoding", e); + } + } + + private static TerraformPath unknown(String raw) { + return new TerraformPath(TerraformPath.Kind.UNKNOWN, null, null, null, + null, null, null, null, null, raw); + } + + public record ParsedRequest(String canonicalPath, String credentialSegment, TerraformPath path) {} +} diff --git a/protocol-terraform/src/main/java/com/github/klboke/kkrepo/protocol/terraform/TerraformVersions.java b/protocol-terraform/src/main/java/com/github/klboke/kkrepo/protocol/terraform/TerraformVersions.java new file mode 100644 index 00000000..1d75777c --- /dev/null +++ b/protocol-terraform/src/main/java/com/github/klboke/kkrepo/protocol/terraform/TerraformVersions.java @@ -0,0 +1,50 @@ +package com.github.klboke.kkrepo.protocol.terraform; + +import java.math.BigInteger; +import java.util.Comparator; +import java.util.List; + +/** Terraform-compatible SemVer precedence (build metadata does not affect ordering). */ +public final class TerraformVersions { + private TerraformVersions() {} + + public static List descending(Iterable versions) { + java.util.ArrayList result = new java.util.ArrayList<>(); + versions.forEach(result::add); + result.sort(comparator().reversed()); + return List.copyOf(result); + } + + public static Comparator comparator() { + return (left, right) -> compare(parse(left), parse(right)); + } + + private static Version parse(String raw) { + String withoutBuild = raw.split("\\+", 2)[0]; + String[] mainAndPre = withoutBuild.split("-", 2); + String[] main = mainAndPre[0].split("\\."); + return new Version(new BigInteger(main[0]), new BigInteger(main[1]), new BigInteger(main[2]), + mainAndPre.length == 1 ? List.of() : List.of(mainAndPre[1].split("\\."))); + } + + private static int compare(Version a, Version b) { + int c = a.major.compareTo(b.major); + if (c == 0) c = a.minor.compareTo(b.minor); + if (c == 0) c = a.patch.compareTo(b.patch); + if (c != 0) return c; + if (a.pre.isEmpty()) return b.pre.isEmpty() ? 0 : 1; + if (b.pre.isEmpty()) return -1; + for (int i = 0; i < Math.min(a.pre.size(), b.pre.size()); i++) { + String x = a.pre.get(i); + String y = b.pre.get(i); + boolean xn = x.matches("[0-9]+"); + boolean yn = y.matches("[0-9]+"); + c = xn && yn ? new BigInteger(x).compareTo(new BigInteger(y)) + : xn ? -1 : yn ? 1 : x.compareTo(y); + if (c != 0) return c; + } + return Integer.compare(a.pre.size(), b.pre.size()); + } + + private record Version(BigInteger major, BigInteger minor, BigInteger patch, List pre) {} +} diff --git a/protocol-terraform/src/test/java/com/github/klboke/kkrepo/protocol/terraform/TerraformPathParserTest.java b/protocol-terraform/src/test/java/com/github/klboke/kkrepo/protocol/terraform/TerraformPathParserTest.java new file mode 100644 index 00000000..94b0ffa7 --- /dev/null +++ b/protocol-terraform/src/test/java/com/github/klboke/kkrepo/protocol/terraform/TerraformPathParserTest.java @@ -0,0 +1,66 @@ +package com.github.klboke.kkrepo.protocol.terraform; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.List; +import org.junit.jupiter.api.Test; + +class TerraformPathParserTest { + private final TerraformPathParser parser = new TerraformPathParser(); + + @Test + void parsesOfficialModuleAndProviderRoutes() { + assertEquals(TerraformPath.Kind.MODULE_VERSIONS, + parser.parse("v1/modules/acme/network/aws/versions").kind()); + assertEquals(TerraformPath.Kind.MODULE_DOWNLOAD, + parser.parse("v1/modules/acme/network/aws/1.2.3/download").kind()); + assertEquals(TerraformPath.Kind.MODULE_ARCHIVE, + parser.parse("v1/modules/acme/network/aws/1.2.3/network.zip").kind()); + TerraformPath upload = parser.parse("v1/providers/acme/cloud/1.2.3/download/linux/amd64"); + assertEquals(TerraformPath.Kind.PROVIDER_DOWNLOAD, upload.kind()); + assertEquals("linux", upload.os()); + assertEquals("amd64", upload.arch()); + assertEquals(TerraformPath.Kind.PROVIDER_ARCHIVE, + parser.parse("v1/providers/acme/cloud/1.2.3/package/linux/terraform-provider-cloud_1.2.3_linux_amd64.zip").kind()); + assertEquals(TerraformPath.Kind.PROVIDER_SHA256SUMS, + parser.parse("v1/providers/acme/cloud/1.2.3/metadata-r2/terraform-provider-cloud_1.2.3_SHA256SUMS").kind()); + } + + @Test + void stripsOneUrlTokenSegmentBeforeProtocolDispatch() { + var module = parser.parseRequestPath("v1/modules/dGVzdDp0ZXN0/acme/network/aws/versions"); + assertEquals("dGVzdDp0ZXN0", module.credentialSegment()); + assertEquals("v1/modules/acme/network/aws/versions", module.canonicalPath()); + var provider = parser.parseRequestPath( + "v1/providers/generic-token/acme/cloud/1.2.3/download/linux/amd64"); + assertEquals("generic-token", provider.credentialSegment()); + assertEquals(TerraformPath.Kind.PROVIDER_DOWNLOAD, provider.path().kind()); + } + + @Test + void stripsProviderVersionsUrlTokenBeforeBuildingProxySuffix() { + var provider = parser.parseRequestPath( + "v1/providers/GenericToken.secret/hashicorp/null/versions"); + assertEquals("GenericToken.secret", provider.credentialSegment()); + assertEquals("v1/providers/hashicorp/null/versions", provider.canonicalPath()); + assertEquals(TerraformPath.Kind.PROVIDER_VERSIONS, provider.path().kind()); + } + + @Test + void rejectsTraversalEncodingAndNonSemanticVersions() { + for (String path : List.of( + "v1/modules/acme/%2Fetc/aws/versions", + "v1/modules/acme/%252Fetc/aws/versions", + "v1/modules/acme/../aws/versions", + "v1/modules/acme/network/aws/latest/download")) { + assertThrows(IllegalArgumentException.class, () -> parser.parse(path), path); + } + } + + @Test + void sortsSemverInsteadOfLexicographically() { + assertEquals(List.of("10.0.0", "2.0.0", "1.0.0", "1.0.0-rc.2", "1.0.0-rc.1"), + TerraformVersions.descending(List.of("1.0.0-rc.1", "2.0.0", "1.0.0", "10.0.0", "1.0.0-rc.2"))); + } +} diff --git a/scripts/ci/live-compat-setup.sh b/scripts/ci/live-compat-setup.sh index 11956e7d..e9d40610 100755 --- a/scripts/ci/live-compat-setup.sh +++ b/scripts/ci/live-compat-setup.sh @@ -680,6 +680,33 @@ ensure_kkrepo_repositories() { "group":{"memberNames":["yum-hosted"]} }' + kkrepo_create_repo "terraform-hosted" '{ + "name":"terraform-hosted", + "recipe":"terraform-hosted", + "online":true, + "blobStoreName":"default", + "strictContentTypeValidation":true, + "hosted":{"writePolicy":"ALLOW_ONCE"} + }' + + kkrepo_create_repo "terraform-proxy" '{ + "name":"terraform-proxy", + "recipe":"terraform-proxy", + "online":true, + "blobStoreName":"default", + "strictContentTypeValidation":true, + "proxy":{"remoteUrl":"https://registry.terraform.io/","contentMaxAgeMinutes":1440,"metadataMaxAgeMinutes":1440,"autoBlock":true} + }' + + kkrepo_create_repo "terraform-group" '{ + "name":"terraform-group", + "recipe":"terraform-group", + "online":true, + "blobStoreName":"default", + "strictContentTypeValidation":true, + "group":{"memberNames":["terraform-hosted","terraform-proxy"]} + }' + kkrepo_create_repo "docker-hosted" "{ \"name\":\"docker-hosted\", \"recipe\":\"docker-hosted\", diff --git a/scripts/ci/run-client-e2e.sh b/scripts/ci/run-client-e2e.sh index 41761b4e..0078abd9 100755 --- a/scripts/ci/run-client-e2e.sh +++ b/scripts/ci/run-client-e2e.sh @@ -998,13 +998,125 @@ test_docker_oci() { fi } +test_terraform() { + need zip + local terraform_013="${TERRAFORM_013_BIN:-}" + local terraform_current="${TERRAFORM_CURRENT_BIN:-}" + if [[ -z "$terraform_013" || ! -x "$terraform_013" ]]; then + log "TERRAFORM_013_BIN must point to an executable Terraform 0.13 binary" + exit 2 + fi + if [[ -z "$terraform_current" || ! -x "$terraform_current" ]]; then + log "TERRAFORM_CURRENT_BIN must point to an executable current stable Terraform binary" + exit 2 + fi + + local dir="$WORK_DIR/terraform" + local fixture_version="1.0.$STAMP" + local fixture_os="${TERRAFORM_E2E_OS:-$(uname -s | tr '[:upper:]' '[:lower:]')}" + local fixture_arch="${TERRAFORM_E2E_ARCH:-$(uname -m)}" + [[ "$fixture_arch" == "x86_64" ]] && fixture_arch="amd64" + [[ "$fixture_arch" == "aarch64" ]] && fixture_arch="arm64" + local module_dir="$dir/module" + local provider_dir="$dir/provider" + local module_zip="$dir/kkrepo-client-e2e-module_${fixture_version}.zip" + local provider_zip="$dir/terraform-provider-fixture_${fixture_version}_${fixture_os}_${fixture_arch}.zip" + local token config + mkdir -p "$module_dir" "$provider_dir" + + cat >"$module_dir/main.tf" <<'EOF' +output "message" { + value = "kkrepo Terraform module client e2e" +} +EOF + run_logged_in terraform-module-archive "$module_dir" zip -q -r "$module_zip" . + + cat >"$provider_dir/terraform-provider-fixture_v$fixture_version" <<'EOF' +#!/usr/bin/env sh +echo "kkrepo Terraform provider fixture is install-only" >&2 +exit 1 +EOF + chmod +x "$provider_dir/terraform-provider-fixture_v$fixture_version" + run_logged_in terraform-provider-archive "$provider_dir" zip -q "$provider_zip" "terraform-provider-fixture_v$fixture_version" + + run_logged terraform-module-upload curl -m 30 --fail-with-body -sS -u "$KKREPO_AUTH" \ + --upload-file "$module_zip" \ + "$KKREPO_URL/repository/terraform-hosted/v1/modules/kkrepo/client-e2e/aws/$fixture_version/$(basename "$module_zip")" + run_logged terraform-provider-upload curl -m 30 --fail-with-body -sS -u "$KKREPO_AUTH" \ + -H "Content-Disposition: attachment; filename=$(basename "$provider_zip")" \ + --upload-file "$provider_zip" \ + "$KKREPO_URL/repository/terraform-hosted/v1/providers/kkrepo/fixture/$fixture_version/download/$fixture_os/$fixture_arch" + + token="$(create_api_key GenericToken "client e2e terraform $STAMP")" + add_redaction_value "$token" + config="$dir/terraform.rc" + cat >"$config" <"$init_dir/main.tf" </dev/null 2>&1 } +terraform_migration_enabled() { + [[ "${NEXUS_COMPAT_IMAGE:-}" == *3.92* && -n "${TERRAFORM_CURRENT_BIN:-}" ]] +} + +source_terraform_available() { + curl -m 20 -fsS \ + -u "$NEXUS_USER:$NEXUS_PASSWORD" \ + "$NEXUS_URL/service/rest/v1/repositories/terraform/hosted/$TERRAFORM_NEXUS_REPOSITORY" >/dev/null 2>&1 +} + warm_composer_proxy_fixture() { local metadata_file dist_file metadata_url dist_url actual_sha1 prefix local -a fields @@ -603,6 +615,130 @@ PY log "Pub fixture verified: $package_name $version sha256=$expected_sha256" } +verify_migrated_terraform_fixture() { + local workdir source_modules target_modules source_providers target_providers + local source_metadata target_metadata fields archive module_version provider_version + local expected_sha downloaded_sha token + workdir="$(mktemp -d "${TMPDIR:-/tmp}/kkrepo-terraform-migration.XXXXXX")" + source_modules="$workdir/source-modules.json" + target_modules="$workdir/target-modules.json" + source_providers="$workdir/source-providers.json" + target_providers="$workdir/target-providers.json" + source_metadata="$workdir/source-provider.json" + target_metadata="$workdir/target-provider.json" + fields="$workdir/provider-fields.txt" + archive="$workdir/provider.zip" + + curl -m 30 -fsS -u "$NEXUS_USER:$NEXUS_PASSWORD" \ + "$NEXUS_URL/repository/$TERRAFORM_NEXUS_REPOSITORY/v1/modules/kkrepo/fixture/aws/versions" \ + >"$source_modules" + curl -m 30 -fsS -u "$(auth)" \ + "$KKREPO_URL/repository/$TERRAFORM_KKREPO_REPOSITORY/v1/modules/kkrepo/fixture/aws/versions" \ + >"$target_modules" + module_version="$(python3 - "$source_modules" "$target_modules" <<'PY' +import json +import sys + +def versions(path): + with open(path, "r", encoding="utf-8") as source: + body = json.load(source) + return [row["version"] for row in body["modules"][0]["versions"]] + +source = versions(sys.argv[1]) +target = versions(sys.argv[2]) +if not source or not set(source).issubset(set(target)): + raise SystemExit(f"migrated Terraform module versions are incomplete: source={source} target={target}") +print(source[0]) +PY +)" + + curl -m 30 -fsS -u "$NEXUS_USER:$NEXUS_PASSWORD" \ + "$NEXUS_URL/repository/$TERRAFORM_NEXUS_REPOSITORY/v1/providers/kkrepo/fixture/versions" \ + >"$source_providers" + curl -m 30 -fsS -u "$(auth)" \ + "$KKREPO_URL/repository/$TERRAFORM_KKREPO_REPOSITORY/v1/providers/kkrepo/fixture/versions" \ + >"$target_providers" + provider_version="$(python3 - "$source_providers" "$target_providers" <<'PY' +import json +import sys + +def versions(path): + with open(path, "r", encoding="utf-8") as source: + body = json.load(source) + return [row["version"] for row in body["versions"]] + +source = versions(sys.argv[1]) +target = versions(sys.argv[2]) +if not source or not set(source).issubset(set(target)): + raise SystemExit(f"migrated Terraform provider versions are incomplete: source={source} target={target}") +print(source[0]) +PY +)" + + curl -m 30 -fsS -u "$NEXUS_USER:$NEXUS_PASSWORD" \ + "$NEXUS_URL/repository/$TERRAFORM_NEXUS_REPOSITORY/v1/providers/kkrepo/fixture/$provider_version/download/linux/amd64" \ + >"$source_metadata" + curl -m 30 -fsS -u "$(auth)" \ + "$KKREPO_URL/repository/$TERRAFORM_KKREPO_REPOSITORY/v1/providers/kkrepo/fixture/$provider_version/download/linux/amd64" \ + >"$target_metadata" + python3 - "$source_metadata" "$target_metadata" "$KKREPO_URL/repository/$TERRAFORM_KKREPO_REPOSITORY/" "$fields" <<'PY' +import json +import sys +from urllib.parse import urljoin + +source_path, target_path, repository_url, output_path = sys.argv[1:5] +with open(source_path, "r", encoding="utf-8") as handle: + source = json.load(handle) +with open(target_path, "r", encoding="utf-8") as handle: + target = json.load(handle) +for field in ("filename", "shasum"): + if source.get(field) != target.get(field): + raise SystemExit(f"migrated Terraform provider {field} changed: {source.get(field)!r} != {target.get(field)!r}") +source_keys = ((source.get("signing_keys") or {}).get("gpg_public_keys") or []) +target_keys = ((target.get("signing_keys") or {}).get("gpg_public_keys") or []) +if not source_keys or not target_keys or source_keys[0].get("key_id") != target_keys[0].get("key_id"): + raise SystemExit("migrated Terraform signing key id changed") +with open(output_path, "w", encoding="utf-8") as output: + output.write(str(target["shasum"]) + "\n") + output.write(urljoin(repository_url, str(target["download_url"])) + "\n") +PY + expected_sha="$(sed -n '1p' "$fields")" + curl -m 30 -fsS -u "$(auth)" "$(sed -n '2p' "$fields")" >"$archive" + downloaded_sha="$(file_sha256 "$archive")" + if [[ "$downloaded_sha" != "$expected_sha" ]]; then + log "Terraform provider checksum mismatch after migration: $downloaded_sha != $expected_sha" + exit 1 + fi + + curl -m 30 -fsS -u "$(auth)" \ + "$KKREPO_URL/internal/repositories/$TERRAFORM_KKREPO_REPOSITORY" >"$workdir/repository.json" + if grep -q 'BEGIN PGP PRIVATE KEY BLOCK\|source-key-passphrase' "$workdir/repository.json"; then + log "Terraform signing secret leaked through migrated repository JSON" + exit 1 + fi + + token="$(printf '%s' "$(auth)" | base64 | tr -d '\r\n')" + mkdir -p "$workdir/module" + cat >"$workdir/terraform.rc" <"$workdir/module/main.tf" <com.github.klboke kkrepo-protocol-composer + + com.github.klboke + kkrepo-protocol-terraform + com.github.klboke kkrepo-protocol-go @@ -143,6 +147,18 @@ org.apache.commons commons-compress + + org.tukaani + xz + + + org.bouncycastle + bcprov-jdk18on + + + org.bouncycastle + bcpg-jdk18on + org.yaml snakeyaml diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/RepositoryContentController.java b/server/src/main/java/com/github/klboke/kkrepo/server/RepositoryContentController.java index 6af10465..09225f1a 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/RepositoryContentController.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/RepositoryContentController.java @@ -13,6 +13,8 @@ import com.github.klboke.kkrepo.protocol.nuget.NugetPaths; import com.github.klboke.kkrepo.protocol.pub.PubPath; import com.github.klboke.kkrepo.protocol.pub.PubPathParser; +import com.github.klboke.kkrepo.protocol.terraform.TerraformPath; +import com.github.klboke.kkrepo.protocol.terraform.TerraformPathParser; import com.github.klboke.kkrepo.server.blob.TempBlobFiles; import com.github.klboke.kkrepo.server.cargo.CargoExceptions; import com.github.klboke.kkrepo.server.cargo.CargoGroupService; @@ -60,6 +62,8 @@ import com.github.klboke.kkrepo.server.rubygems.RubygemsService; import com.github.klboke.kkrepo.server.security.AuthenticatedSubject; import com.github.klboke.kkrepo.server.security.ForwardedHeaderPolicy; +import com.github.klboke.kkrepo.server.security.RepositorySecurityFilter; +import com.github.klboke.kkrepo.server.terraform.TerraformService; import com.github.klboke.kkrepo.server.yum.YumService; import jakarta.servlet.ServletException; import jakarta.servlet.http.HttpServletRequest; @@ -124,6 +128,7 @@ public class RepositoryContentController { private final ComposerHostedService composerHosted; private final ComposerProxyService composerProxy; private final ComposerGroupService composerGroup; + private final TerraformService terraform; private final NugetService nuget; private final RubygemsService rubygems; private final YumService yum; @@ -137,6 +142,7 @@ public class RepositoryContentController { private final CargoPathParser cargoParser = new CargoPathParser(); private final PubPathParser pubParser = new PubPathParser(); private final ComposerPathParser composerParser = new ComposerPathParser(); + private final TerraformPathParser terraformParser = new TerraformPathParser(); private final MavenPartialFetchSupport partialFetch = new MavenPartialFetchSupport(); private final PypiPartialFetchSupport pypiPartialFetch = new PypiPartialFetchSupport(); @@ -158,7 +164,8 @@ public RepositoryContentController(RepositoryRuntimeRegistry registry, YumService yum, RawHostedService rawHosted, RawProxyService rawProxy, RawGroupService rawGroup, ObjectMapper objectMapper, - ForwardedHeaderPolicy forwardedHeaderPolicy) { + ForwardedHeaderPolicy forwardedHeaderPolicy, + TerraformService terraform) { this.registry = registry; this.hosted = hosted; this.proxy = proxy; @@ -185,6 +192,7 @@ public RepositoryContentController(RepositoryRuntimeRegistry registry, this.composerHosted = composerHosted; this.composerProxy = composerProxy; this.composerGroup = composerGroup; + this.terraform = terraform; this.nuget = nuget; this.rubygems = rubygems; this.yum = yum; @@ -195,6 +203,29 @@ public RepositoryContentController(RepositoryRuntimeRegistry registry, this.forwardedHeaderPolicy = forwardedHeaderPolicy; } + /** Backward-compatible full constructor retained for focused unit tests. */ + public RepositoryContentController(RepositoryRuntimeRegistry registry, + MavenHostedService hosted, MavenProxyService proxy, MavenGroupService group, + GoProxyService goProxy, GoGroupService goGroup, + HelmHostedService helmHosted, HelmProxyService helmProxy, + MavenHtmlListingService htmlListing, + NpmHostedService npmHosted, NpmProxyService npmProxy, NpmGroupService npmGroup, + NpmSearchService npmSearch, NpmTokenService npmToken, + PypiHostedService pypiHosted, PypiProxyService pypiProxy, PypiGroupService pypiGroup, + CargoHostedService cargoHosted, CargoProxyService cargoProxy, CargoGroupService cargoGroup, + PubHostedService pubHosted, PubProxyService pubProxy, PubGroupService pubGroup, + ComposerHostedService composerHosted, ComposerProxyService composerProxy, + ComposerGroupService composerGroup, + NugetService nuget, RubygemsService rubygems, YumService yum, + RawHostedService rawHosted, RawProxyService rawProxy, RawGroupService rawGroup, + ObjectMapper objectMapper, ForwardedHeaderPolicy forwardedHeaderPolicy) { + this(registry, hosted, proxy, group, goProxy, goGroup, helmHosted, helmProxy, htmlListing, + npmHosted, npmProxy, npmGroup, npmSearch, npmToken, pypiHosted, pypiProxy, pypiGroup, + cargoHosted, cargoProxy, cargoGroup, pubHosted, pubProxy, pubGroup, + composerHosted, composerProxy, composerGroup, nuget, rubygems, yum, + rawHosted, rawProxy, rawGroup, objectMapper, forwardedHeaderPolicy, null); + } + /** Backward-compatible constructor used by focused controller unit tests. */ public RepositoryContentController(RepositoryRuntimeRegistry registry, MavenHostedService hosted, MavenProxyService proxy, MavenGroupService group, @@ -218,7 +249,7 @@ public RepositoryContentController(RepositoryRuntimeRegistry registry, cargoHosted, cargoProxy, cargoGroup, pubHosted, pubProxy, pubGroup, null, null, null, - nuget, rubygems, yum, rawHosted, rawProxy, rawGroup, objectMapper, forwardedHeaderPolicy); + nuget, rubygems, yum, rawHosted, rawProxy, rawGroup, objectMapper, forwardedHeaderPolicy, null); } @GetMapping("/**") @@ -264,6 +295,12 @@ public ResponseEntity head(@PathVariable("name") String name, HttpServletR MavenResponse resp = dispatchComposerGet(runtime, path, request, true); return toHeadResponse(resp, request); } + if (runtime.format() == RepositoryFormat.TERRAFORM) { + TerraformPath path = terraformParser.parse(extractRepositoryPath(name, request, true)); + return toHeadResponse(terraform.get( + runtime, path, repositoryBaseUrl(request, runtime.name()), + terraformUrlTokenSegment(request), true), request); + } if (runtime.format() == RepositoryFormat.PYPI) { String raw = extractRepositoryPath(name, request, true); PypiResponse resp = isDirectoryPath(raw) @@ -355,6 +392,15 @@ public ResponseEntity put( } return toByteArrayResponse(resp); } + if (runtime.format() == RepositoryFormat.TERRAFORM) { + TerraformPath path = terraformParser.parse(extractRepositoryPath(name, request, true)); + MavenResponse resp; + try (InputStream body = request.getInputStream()) { + resp = terraform.put(runtime, path, body, contentType, + request.getHeader(HttpHeaders.CONTENT_DISPOSITION), userId, request.getRemoteAddr()); + } + return ResponseEntity.status(resp.status()).build(); + } if (runtime.format() == RepositoryFormat.NUGET) { String raw = extractRepositoryPath(name, request, true); MavenResponse resp; @@ -439,6 +485,9 @@ public ResponseEntity delete(@PathVariable("name") String name, HttpServletRe MavenResponse resp = cargoHosted.yank(runtime, path.crateName(), path.version(), true); return toByteArrayResponse(resp); } + if (runtime.format() == RepositoryFormat.TERRAFORM) { + throw new MavenExceptions.MethodNotAllowed("Terraform client repository paths do not support DELETE"); + } if (runtime.format() == RepositoryFormat.NUGET) { String raw = extractRepositoryPath(name, request, true); MavenResponse resp = nuget.deletePackage(runtime, raw); @@ -591,6 +640,13 @@ private ResponseEntity serveBody( MavenResponse resp = dispatchComposerGet(runtime, path, request, headOnly); return toStreamingResponse(resp, request, false); } + if (runtime.format() == RepositoryFormat.TERRAFORM) { + TerraformPath path = terraformParser.parse(extractRepositoryPath(name, request, true)); + MavenResponse resp = terraform.get( + runtime, path, repositoryBaseUrl(request, runtime.name()), + terraformUrlTokenSegment(request), headOnly); + return toStreamingResponse(resp, request, false); + } if (runtime.format() == RepositoryFormat.PYPI) { String raw = extractRepositoryPath(name, request, true); boolean directory = isDirectoryPath(raw); @@ -1026,6 +1082,8 @@ private String extractMavenPath(String name, HttpServletRequest request) { } private String extractRepositoryPath(String name, HttpServletRequest request, boolean allowEmpty) { + Object normalized = request.getAttribute(RepositorySecurityFilter.NORMALIZED_REPOSITORY_PATH_ATTRIBUTE); + if (normalized instanceof String path) return path; String uri = request.getRequestURI(); String root = request.getContextPath() + "/repository/" + name; if (uri.equals(root)) { @@ -1044,6 +1102,8 @@ private String extractRepositoryPath(String name, HttpServletRequest request, bo } private String extractRepositoryPath(String name, HttpServletRequest request) { + Object normalized = request.getAttribute(RepositorySecurityFilter.NORMALIZED_REPOSITORY_PATH_ATTRIBUTE); + if (normalized instanceof String path) return path; String uri = request.getRequestURI(); String base = request.getContextPath() + "/repository/" + name; if (uri.equals(base)) { @@ -1066,6 +1126,11 @@ private String repositoryBaseUrl(HttpServletRequest request, String name) { return serverBaseUrl(request) + request.getContextPath() + "/repository/" + name; } + private static String terraformUrlTokenSegment(HttpServletRequest request) { + Object value = request.getAttribute(RepositorySecurityFilter.TERRAFORM_URL_TOKEN_SEGMENT_ATTRIBUTE); + return value instanceof String token && !token.isBlank() ? token : null; + } + private String serverBaseUrl(HttpServletRequest request) { return forwardedHeaderPolicy.serverBaseUrl(request); } diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/maven/MavenErrorAdvice.java b/server/src/main/java/com/github/klboke/kkrepo/server/maven/MavenErrorAdvice.java index 2ba920fc..bb5be02c 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/maven/MavenErrorAdvice.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/maven/MavenErrorAdvice.java @@ -38,6 +38,11 @@ public ResponseEntity> writeDenied(MavenExceptions.WritePoli return body(HttpStatus.BAD_REQUEST, e.getMessage()); } + @ExceptionHandler(MavenExceptions.BadRequestException.class) + public ResponseEntity> badRequest(MavenExceptions.BadRequestException e) { + return body(HttpStatus.BAD_REQUEST, e.getMessage()); + } + @ExceptionHandler(MavenExceptions.MethodNotAllowed.class) public ResponseEntity> method(MavenExceptions.MethodNotAllowed e) { return body(HttpStatus.METHOD_NOT_ALLOWED, e.getMessage()); diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/maven/MavenExceptions.java b/server/src/main/java/com/github/klboke/kkrepo/server/maven/MavenExceptions.java index d9f0f0c8..c713c690 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/maven/MavenExceptions.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/maven/MavenExceptions.java @@ -20,6 +20,11 @@ public static class WritePolicyDenied extends RuntimeException { public WritePolicyDenied(String message) { super(message); } } + public static class BadRequestException extends RuntimeException { + public BadRequestException(String message) { super(message); } + public BadRequestException(String message, Throwable cause) { super(message, cause); } + } + public static class MethodNotAllowed extends RuntimeException { public MethodNotAllowed(String message) { super(message); } } diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/metrics/RepositoryRequestMetricsFilter.java b/server/src/main/java/com/github/klboke/kkrepo/server/metrics/RepositoryRequestMetricsFilter.java index 9ad16e4f..6fc087d3 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/metrics/RepositoryRequestMetricsFilter.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/metrics/RepositoryRequestMetricsFilter.java @@ -74,7 +74,8 @@ protected void doFilterInternal( String repository = record == null ? "unknown" : record.name(); String format = record == null ? "unknown" : KkRepoMetrics.format(record.format()); String type = record == null ? "unknown" : KkRepoMetrics.type(record.type()); - String operation = operation(target, record == null ? null : record.format(), request.getMethod()); + Target effectiveTarget = effectiveTarget(request, target, record); + String operation = operation(effectiveTarget, record == null ? null : record.format(), request.getMethod()); int status = failure == null ? response.getStatus() : statusFor(failure, response.getStatus()); metrics.recordRepositoryRequest( repository, @@ -87,7 +88,7 @@ protected void doFilterInternal( contentLength(response), failure, sample); - logNonSuccessRequest(request, target, repository, format, type, operation, status, failure); + logNonSuccessRequest(request, effectiveTarget, repository, format, type, operation, status, failure); } } @@ -111,7 +112,7 @@ private void logNonSuccessRequest( + "userAgent={} failure={}", status, request.getMethod(), - stripContextPath(request), + loggedUri(request, target, format), target.path(), requestParameters(request), repository, @@ -175,6 +176,28 @@ private static RepositoryRecord repositoryRecord(HttpServletRequest request) { return value instanceof RepositoryRecord record ? record : null; } + private static Target effectiveTarget( + HttpServletRequest request, Target target, RepositoryRecord repository) { + if (repository == null || repository.format() != RepositoryFormat.TERRAFORM) { + return target; + } + Object normalized = request.getAttribute(RepositorySecurityFilter.NORMALIZED_REPOSITORY_PATH_ATTRIBUTE); + if (normalized instanceof String path && !path.isBlank()) { + return new Target(target.repository(), path, target.route()); + } + // A rejected Terraform URL may still contain a credential segment. If the security filter + // could not canonicalize it, discard the entire path rather than risk logging the token. + return new Target(target.repository(), "", target.route()); + } + + private static String loggedUri( + HttpServletRequest request, Target target, String format) { + if (!"terraform".equals(format) || !"repository".equals(target.route())) { + return stripContextPath(request); + } + return "/repository/" + target.repository() + "/" + target.path(); + } + private static Target target(HttpServletRequest request) { String method = request.getMethod() == null ? "" : request.getMethod().toUpperCase(Locale.ROOT); String uri = stripContextPath(request); @@ -246,10 +269,30 @@ private static String operation(Target target, RepositoryFormat format, String m case RUBYGEMS -> rubygemsOperation(path, normalizedMethod); case YUM -> yumOperation(path, normalizedMethod); case DOCKER -> dockerOperation(path, normalizedMethod); + case TERRAFORM -> terraformOperation(path, normalizedMethod); case RAW -> rawOperation(normalizedMethod); }; } + private static String terraformOperation(String path, String method) { + String redacted = path == null ? "" : path; + if (redacted.startsWith("v1/modules/")) { + if (redacted.endsWith("/versions")) return "terraform_module_versions"; + if (redacted.endsWith("/download")) return "terraform_module_download_metadata"; + return "PUT".equals(method) ? "terraform_module_upload" : "terraform_module_archive"; + } + if (redacted.startsWith("v1/providers/")) { + if (redacted.endsWith("/versions")) return "terraform_provider_versions"; + if (redacted.contains("/download/")) { + return "PUT".equals(method) ? "terraform_provider_upload" : "terraform_provider_download_metadata"; + } + if (redacted.endsWith("_SHA256SUMS.sig")) return "terraform_provider_signature"; + if (redacted.endsWith("_SHA256SUMS")) return "terraform_provider_shasums"; + return "terraform_provider_archive"; + } + return "terraform_repository"; + } + private static String composerOperation(String path, String method) { if (COMPOSER_PATHS.parse(path).kind() == ComposerPath.Kind.DIST) return "composer_dist"; if (path.equals("packages.json") || path.startsWith("p2/") diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/migration/NexusMigrationController.java b/server/src/main/java/com/github/klboke/kkrepo/server/migration/NexusMigrationController.java index f06cbed0..6a06e994 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/migration/NexusMigrationController.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/migration/NexusMigrationController.java @@ -12,6 +12,7 @@ import com.github.klboke.kkrepo.persistence.jdbc.api.MigrationJobDao; import com.github.klboke.kkrepo.persistence.jdbc.api.RepositoryDao; import com.github.klboke.kkrepo.persistence.jdbc.api.SecurityDao; +import com.github.klboke.kkrepo.persistence.jdbc.api.TerraformRegistryDao; import com.github.klboke.kkrepo.persistence.jdbc.api.model.BlobStoreRecord; import com.github.klboke.kkrepo.server.docker.DockerConnectorRuntime; import com.github.klboke.kkrepo.server.maven.BlobStorageRegistry; @@ -66,6 +67,7 @@ public class NexusMigrationController { private final ApiKeyAuthCache apiKeyAuthCache; private final BasicAuthCache basicAuthCache; private final DockerConnectorRuntime dockerConnectorRuntime; + private final TerraformRegistryDao terraformRegistry; public NexusMigrationController( ObjectMapper objectMapper, @@ -84,7 +86,8 @@ public NexusMigrationController( SecurityAuthorizationCache securityAuthorizationCache, ApiKeyAuthCache apiKeyAuthCache, BasicAuthCache basicAuthCache, - DockerConnectorRuntime dockerConnectorRuntime) { + DockerConnectorRuntime dockerConnectorRuntime, + TerraformRegistryDao terraformRegistry) { this.objectMapper = objectMapper; this.blobStoreDao = blobStoreDao; this.repositoryDao = repositoryDao; @@ -102,6 +105,7 @@ public NexusMigrationController( this.apiKeyAuthCache = apiKeyAuthCache; this.basicAuthCache = basicAuthCache; this.dockerConnectorRuntime = dockerConnectorRuntime; + this.terraformRegistry = terraformRegistry; } @PostMapping("/preflight") @@ -256,7 +260,8 @@ NexusApiMigrationService service() { repositoryDao, securityDao, migrationJobDao, - new SecurityDaoMigrationWriter(securityDao)); + new SecurityDaoMigrationWriter(securityDao), + terraformRegistry); } private NexusMigrationRequest toRequest(NexusMigrationCommand command, boolean dryRun) { diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/migration/RepositoryDataMigrationPaths.java b/server/src/main/java/com/github/klboke/kkrepo/server/migration/RepositoryDataMigrationPaths.java index 678ab072..fd165abe 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/migration/RepositoryDataMigrationPaths.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/migration/RepositoryDataMigrationPaths.java @@ -3,6 +3,7 @@ import com.github.klboke.kkrepo.core.RepositoryFormat; import com.github.klboke.kkrepo.protocol.maven.path.MavenPath; import com.github.klboke.kkrepo.server.pub.PubRepositoryDataMigrationWriter; +import com.github.klboke.kkrepo.server.terraform.TerraformRepositoryDataMigrationWriter; import java.util.Locale; final class RepositoryDataMigrationPaths { @@ -26,6 +27,9 @@ static boolean shouldDiscoverAsset(RepositoryFormat format, String path) { if (format == RepositoryFormat.PUB) { return PubRepositoryDataMigrationWriter.isMigratablePubPath(path); } + if (format == RepositoryFormat.TERRAFORM) { + return TerraformRepositoryDataMigrationWriter.isMigratableTerraformPath(path); + } return true; } diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/migration/RepositoryDataMigrationWorker.java b/server/src/main/java/com/github/klboke/kkrepo/server/migration/RepositoryDataMigrationWorker.java index bc701eb8..5148465c 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/migration/RepositoryDataMigrationWorker.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/migration/RepositoryDataMigrationWorker.java @@ -310,7 +310,8 @@ static boolean shouldMigrateSourceAsset(RepositoryFormat format, String path) { if (format == RepositoryFormat.CARGO && isCargoDynamicConfig(path)) { return false; } - if (format == RepositoryFormat.PUB && !RepositoryDataMigrationPaths.shouldDiscoverAsset(format, path)) { + if ((format == RepositoryFormat.PUB || format == RepositoryFormat.TERRAFORM) + && !RepositoryDataMigrationPaths.shouldDiscoverAsset(format, path)) { return false; } return true; diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/migration/RepositoryDataMigrationWriter.java b/server/src/main/java/com/github/klboke/kkrepo/server/migration/RepositoryDataMigrationWriter.java index e02497a5..e31d89db 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/migration/RepositoryDataMigrationWriter.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/migration/RepositoryDataMigrationWriter.java @@ -35,6 +35,7 @@ import com.github.klboke.kkrepo.server.docker.DockerManifestParser; import com.github.klboke.kkrepo.server.maven.BlobStorageRegistry; import com.github.klboke.kkrepo.server.pub.PubRepositoryDataMigrationWriter; +import com.github.klboke.kkrepo.server.terraform.TerraformRepositoryDataMigrationWriter; import com.github.klboke.kkrepo.server.transaction.TransientTransactionRetry; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -56,6 +57,7 @@ import java.util.Optional; import java.util.OptionalLong; import org.springframework.stereotype.Component; +import org.springframework.beans.factory.annotation.Autowired; @Component class RepositoryDataMigrationWriter { @@ -71,6 +73,7 @@ class RepositoryDataMigrationWriter { private final DockerRegistryDao dockerRegistryDao; private final DockerManifestParser dockerManifestParser; private final PubRepositoryDataMigrationWriter pubMigrationWriter; + private final TerraformRepositoryDataMigrationWriter terraformMigrationWriter; private final TransientTransactionRetry transactionRetry; private final MavenPathParser mavenPathParser = new MavenPathParser(); @@ -85,6 +88,33 @@ class RepositoryDataMigrationWriter { DockerManifestParser dockerManifestParser, PubRepositoryDataMigrationWriter pubMigrationWriter, TransientTransactionRetry transactionRetry) { + this( + repositoryDao, + componentDao, + assetDao, + browseNodeDao, + blobStorageRegistry, + indexRebuildDao, + dockerRegistryDao, + dockerManifestParser, + pubMigrationWriter, + null, + transactionRetry); + } + + @Autowired + RepositoryDataMigrationWriter( + RepositoryDao repositoryDao, + ComponentDao componentDao, + AssetDao assetDao, + BrowseNodeDao browseNodeDao, + BlobStorageRegistry blobStorageRegistry, + RepositoryIndexRebuildDao indexRebuildDao, + DockerRegistryDao dockerRegistryDao, + DockerManifestParser dockerManifestParser, + PubRepositoryDataMigrationWriter pubMigrationWriter, + TerraformRepositoryDataMigrationWriter terraformMigrationWriter, + TransientTransactionRetry transactionRetry) { this.repositoryDao = repositoryDao; this.componentDao = componentDao; this.assetDao = assetDao; @@ -94,6 +124,7 @@ class RepositoryDataMigrationWriter { this.dockerRegistryDao = dockerRegistryDao; this.dockerManifestParser = dockerManifestParser; this.pubMigrationWriter = pubMigrationWriter; + this.terraformMigrationWriter = terraformMigrationWriter; this.transactionRetry = transactionRetry; } @@ -114,6 +145,15 @@ WriteResult write(long targetRepositoryId, RepositoryDataMigrationAssetRecord so return new WriteResult( migrated.componentId(), migrated.assetId(), migrated.assetBlobId(), migrated.assetBlobObjectKey()); } + if (repository.format() == RepositoryFormat.TERRAFORM) { + if (terraformMigrationWriter == null) { + throw new IllegalStateException("Terraform migration writer is not configured"); + } + TerraformRepositoryDataMigrationWriter.MigratedAsset migrated = terraformMigrationWriter.write( + repository, source, body, responseContentType, validateSize); + return new WriteResult( + migrated.componentId(), migrated.assetId(), migrated.assetBlobId(), migrated.assetBlobObjectKey()); + } DigestedUpload upload = uploadWithDigests(repository, storage, source, body, validateSize); Map checksumUploads = new LinkedHashMap<>(); Map checksumResults = new LinkedHashMap<>(); @@ -746,6 +786,7 @@ private static String assetKind(RepositoryFormat format, RepositoryDataMigration case PUB -> source.sourcePath().endsWith(".tar.gz") ? "archive" : "metadata"; case COMPOSER -> COMPOSER_PATHS.parse(source.sourcePath()).kind() == ComposerPath.Kind.DIST ? "composer-dist" : "composer-metadata"; + case TERRAFORM -> "terraform-asset"; case RAW -> "asset"; }; } diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/repositories/RepositoryService.java b/server/src/main/java/com/github/klboke/kkrepo/server/repositories/RepositoryService.java index 7be1e55e..4bace031 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/repositories/RepositoryService.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/repositories/RepositoryService.java @@ -7,6 +7,7 @@ import com.github.klboke.kkrepo.persistence.jdbc.api.BlobStoreDao; import com.github.klboke.kkrepo.persistence.jdbc.api.RepositoryDao; import com.github.klboke.kkrepo.persistence.jdbc.api.SecurityDao; +import com.github.klboke.kkrepo.persistence.jdbc.api.TerraformRegistryDao; import com.github.klboke.kkrepo.persistence.jdbc.api.model.BlobStoreRecord; import com.github.klboke.kkrepo.persistence.jdbc.api.model.RepositoryRecord; import com.github.klboke.kkrepo.server.docker.DockerConnectorRuntime; @@ -66,6 +67,7 @@ public class RepositoryService { private final NexusLikeCacheController cacheController; private final RepositoryCatalogCache repositoryCatalogCache; private final DockerConnectorRuntime dockerConnectorRuntime; + private final TerraformRegistryDao terraformRegistry; private final String urlPrefix; private final int serverPort; private final int managementPort; @@ -86,6 +88,7 @@ public RepositoryService( NexusLikeCacheController cacheController, RepositoryCatalogCache repositoryCatalogCache, DockerConnectorRuntime dockerConnectorRuntime, + TerraformRegistryDao terraformRegistry, @Value("${kkrepo.compatibility.repository-url-prefix:/repository}") String urlPrefix, @Value("${server.port:8080}") int serverPort, @Value("${management.server.port:${server.port:8080}}") int managementPort) { @@ -103,6 +106,7 @@ public RepositoryService( this.cacheController = cacheController; this.repositoryCatalogCache = repositoryCatalogCache; this.dockerConnectorRuntime = dockerConnectorRuntime; + this.terraformRegistry = terraformRegistry; this.urlPrefix = urlPrefix; this.serverPort = serverPort; this.managementPort = managementPort; @@ -116,7 +120,7 @@ public RepositoryService( String urlPrefix) { this(repositoryDao, blobStoreDao, securityDao, runtimeRegistry, null, null, null, OutboundRequestPolicy.allowPrivateForTests(), null, null, null, null, null, null, - urlPrefix, 8080, 8080); + null, urlPrefix, 8080, 8080); } public RepositoryService( @@ -128,7 +132,7 @@ public RepositoryService( String urlPrefix) { this(repositoryDao, blobStoreDao, securityDao, runtimeRegistry, null, null, null, OutboundRequestPolicy.allowPrivateForTests(), null, null, null, cacheController, null, null, - urlPrefix, 8080, 8080); + null, urlPrefix, 8080, 8080); } RepositoryService( @@ -140,7 +144,7 @@ public RepositoryService( int serverPort, int managementPort) { this(repositoryDao, blobStoreDao, securityDao, runtimeRegistry, null, null, - null, OutboundRequestPolicy.allowPrivateForTests(), null, null, null, null, null, null, urlPrefix, + null, OutboundRequestPolicy.allowPrivateForTests(), null, null, null, null, null, null, null, urlPrefix, serverPort, managementPort); } @@ -321,10 +325,12 @@ public RepositoryView update(String name, UpdateCommand command) { invalidateNpmGroupAfterCommit(existing.format(), existing.id()); invalidatePypiGroupAfterCommit(existing.format(), existing.id()); invalidateGroupMemberGroupAfterCommit(existing.format(), existing.id()); + invalidateTerraformGroupBindings(existing.format(), existing.id()); } else if (recipe.type() != RepositoryType.GROUP) { invalidateNpmMemberAfterCommit(existing.format(), existing.id()); invalidatePypiMemberAfterCommit(existing.format(), existing.id()); invalidateGroupMemberMemberAfterCommit(existing.format(), existing.id()); + invalidateTerraformContainingGroupBindings(existing.format(), existing.id(), new HashSet<>()); } invalidateRuntimeCache(existing.id(), name); @@ -375,6 +381,7 @@ public RepositoryView replaceMembers(String name, List memberNames) { invalidateNpmGroupAfterCommit(existing.format(), existing.id()); invalidatePypiGroupAfterCommit(existing.format(), existing.id()); invalidateGroupMemberGroupAfterCommit(existing.format(), existing.id()); + invalidateTerraformGroupBindings(existing.format(), existing.id()); runtimeRegistry.invalidate(name); invalidateRepositoryCacheTokensAfterCommit(existing.id()); refreshRepositoryCatalogAfterCommit(); @@ -409,6 +416,28 @@ private void invalidateContainingRuntimeCaches(long repositoryId, Set visi } } + private void invalidateTerraformGroupBindings(RepositoryFormat format, long groupRepositoryId) { + if (format != RepositoryFormat.TERRAFORM || terraformRegistry == null) { + return; + } + terraformRegistry.deleteSourceBindings(groupRepositoryId); + invalidateTerraformContainingGroupBindings(format, groupRepositoryId, new HashSet<>()); + } + + private void invalidateTerraformContainingGroupBindings( + RepositoryFormat format, long repositoryId, Set visited) { + if (format != RepositoryFormat.TERRAFORM || terraformRegistry == null) { + return; + } + for (RepositoryRecord group : repositoryDao.listGroupsContaining(repositoryId)) { + if (group.id() == null || !visited.add(group.id())) { + continue; + } + terraformRegistry.deleteSourceBindings(group.id()); + invalidateTerraformContainingGroupBindings(format, group.id(), visited); + } + } + private DockerSettings normalizeDocker(DockerSettings settings) { if (settings == null) { return new DockerSettings(false, null, null); @@ -755,9 +784,12 @@ private static HostedSettings mergeHosted(RepositoryRecord existing, HostedSetti } private ProxySettings requireProxy(ProxySettings settings, RepositoryFormat format) { - if (settings == null && (format == RepositoryFormat.PUB || format == RepositoryFormat.COMPOSER)) { + if (settings == null && (format == RepositoryFormat.PUB + || format == RepositoryFormat.COMPOSER || format == RepositoryFormat.TERRAFORM)) { settings = new ProxySettings( - format == RepositoryFormat.PUB ? "https://pub.dev/" : "https://repo.packagist.org/", + format == RepositoryFormat.PUB ? "https://pub.dev/" + : format == RepositoryFormat.COMPOSER ? "https://repo.packagist.org/" + : "https://registry.terraform.io/", null, null, null); } if (settings == null) { @@ -789,6 +821,14 @@ private ProxySettings requireProxy(ProxySettings settings, RepositoryFormat form settings.remoteBearerToken(), settings.remoteBearerTokenConfigured()); } + if (format == RepositoryFormat.TERRAFORM + && (settings.remoteUrl() == null || settings.remoteUrl().isBlank())) { + settings = new ProxySettings( + "https://registry.terraform.io/", + settings.contentMaxAgeMinutes(), settings.metadataMaxAgeMinutes(), settings.autoBlock(), + settings.remoteUsername(), settings.remotePassword(), settings.remotePasswordConfigured(), + settings.remoteBearerToken(), settings.remoteBearerTokenConfigured()); + } validateProxy(settings); return settings; } @@ -1000,7 +1040,8 @@ && groupWouldCreateCycle(member, targetGroupName, format, visited)) { } private static boolean supportsNestedGroups(RepositoryFormat format) { - return format == RepositoryFormat.PUB || format == RepositoryFormat.COMPOSER; + return format == RepositoryFormat.PUB || format == RepositoryFormat.COMPOSER + || format == RepositoryFormat.TERRAFORM; } private static String stringValue(Object value, String fallback) { diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/security/RepositorySecurityFilter.java b/server/src/main/java/com/github/klboke/kkrepo/server/security/RepositorySecurityFilter.java index 07b536f5..70919052 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/security/RepositorySecurityFilter.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/security/RepositorySecurityFilter.java @@ -10,6 +10,7 @@ import com.github.klboke.kkrepo.persistence.jdbc.api.model.RepositoryRecord; import com.github.klboke.kkrepo.protocol.pub.PubPath; import com.github.klboke.kkrepo.protocol.pub.PubPathParser; +import com.github.klboke.kkrepo.protocol.terraform.TerraformPathParser; import com.github.klboke.kkrepo.server.npm.NpmTokenService; import jakarta.servlet.FilterChain; import jakarta.servlet.ServletException; @@ -32,7 +33,12 @@ public class RepositorySecurityFilter extends OncePerRequestFilter { static final int FILTER_ORDER = SessionRepositoryFilter.DEFAULT_ORDER + 20; public static final String REPOSITORY_RECORD_ATTRIBUTE = RepositorySecurityFilter.class.getName() + ".REPOSITORY_RECORD"; + public static final String NORMALIZED_REPOSITORY_PATH_ATTRIBUTE = + RepositorySecurityFilter.class.getName() + ".NORMALIZED_REPOSITORY_PATH"; + public static final String TERRAFORM_URL_TOKEN_SEGMENT_ATTRIBUTE = + RepositorySecurityFilter.class.getName() + ".TERRAFORM_URL_TOKEN_SEGMENT"; private static final PubPathParser PUB_PATH_PARSER = new PubPathParser(); + private static final TerraformPathParser TERRAFORM_PATH_PARSER = new TerraformPathParser(); private final SecurityAuthenticationService authenticationService; private final AccessDecisionService accessDecisionService; private final RepositoryDao repositoryDao; @@ -70,6 +76,25 @@ protected void doFilterInternal( return; } request.setAttribute(REPOSITORY_RECORD_ATTRIBUTE, repository.get()); + String terraformUrlToken = null; + if (repository.get().format() == RepositoryFormat.TERRAFORM) { + try { + String presentedPath = target.path(); + TerraformPathParser.ParsedRequest parsed = TERRAFORM_PATH_PARSER.parseRequestPath(target.path()); + target = target.withPath(parsed.canonicalPath()); + terraformUrlToken = parsed.credentialSegment(); + request.setAttribute(NORMALIZED_REPOSITORY_PATH_ATTRIBUTE, parsed.canonicalPath()); + if (terraformUrlToken != null) { + String[] segments = presentedPath.split("/", 4); + if (segments.length >= 3) { + request.setAttribute(TERRAFORM_URL_TOKEN_SEGMENT_ATTRIBUTE, segments[2]); + } + } + } catch (IllegalArgumentException e) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST, "Invalid Terraform repository path"); + return; + } + } if (target.npmTokenRoute() && repository.get().format() == RepositoryFormat.NPM) { filterChain.doFilter(request, response); return; @@ -78,12 +103,14 @@ protected void doFilterInternal( filterChain.doFilter(request, response); return; } - Optional authenticated = switch (repository.get().format()) { + Optional authenticated = terraformUrlToken == null + ? switch (repository.get().format()) { case CARGO -> authenticationService.authenticateCargo(request); case PUB -> authenticationService.authenticatePub(request); case RUBYGEMS -> authenticationService.authenticateRubygems(request); default -> authenticationService.authenticate(request); - }; + } + : authenticationService.authenticateTerraformUrlToken(terraformUrlToken); if (authenticated.isEmpty()) { authenticated = target.readOnly(repository.get().format()) ? authenticationService.authenticateAnonymous() @@ -200,6 +227,9 @@ private static List actionsForRepository(String method, String if (format == RepositoryFormat.PUB && isPubPublishRoute(method, path)) { return List.of(PermissionAction.ADD); } + if (format == RepositoryFormat.TERRAFORM && "PUT".equalsIgnoreCase(method)) { + return List.of(PermissionAction.ADD, PermissionAction.EDIT); + } if (format == RepositoryFormat.CARGO && isCargoYankRoute(method, path)) { return List.of(PermissionAction.EDIT); } @@ -310,5 +340,9 @@ private boolean readOnly(RepositoryFormat format) { .allMatch(action -> action == PermissionAction.BROWSE || action == PermissionAction.READ); } + private RepositoryRequest withPath(String normalizedPath) { + return new RepositoryRequest(repository, normalizedPath, method, fixedActions, npmTokenRoute); + } + } } diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/security/SecurityAuthenticationService.java b/server/src/main/java/com/github/klboke/kkrepo/server/security/SecurityAuthenticationService.java index c2d0edde..edca3604 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/security/SecurityAuthenticationService.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/security/SecurityAuthenticationService.java @@ -179,6 +179,25 @@ public Optional authenticateRubygems(HttpServletRequest re return authenticate(request); } + /** Authenticates the single decoded credential segment used by Nexus Terraform repository URLs. */ + @Transactional + public Optional authenticateTerraformUrlToken(String token) { + if (token == null || token.isBlank()) return Optional.empty(); + Optional apiKey = apiKeyAuthCache == null + ? resolveApiKey(ApiKeyTokenCandidate.fromPresentedToken(token)) + : apiKeyAuthCache.find("terraform:" + token, + () -> resolveApiKey(ApiKeyTokenCandidate.fromPresentedToken(token))); + if (apiKey.isPresent()) return apiKey; + try { + String decoded = new String(java.util.Base64.getDecoder().decode(token), StandardCharsets.UTF_8); + int colon = decoded.indexOf(':'); + if (colon > 0) return authenticateBasic(decoded.substring(0, colon), decoded.substring(colon + 1)); + } catch (IllegalArgumentException ignored) { + // GenericToken/API-key URL tokens are not necessarily base64 Basic credentials. + } + return Optional.empty(); + } + @Transactional public Optional authenticateCredentials(String username, String password) { if (username == null || username.isBlank() || password == null) { diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformArchiveInspector.java b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformArchiveInspector.java new file mode 100644 index 00000000..5859632f --- /dev/null +++ b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformArchiveInspector.java @@ -0,0 +1,179 @@ +package com.github.klboke.kkrepo.server.terraform; + +import com.github.klboke.kkrepo.server.maven.MavenExceptions; +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.HashSet; +import java.util.Locale; +import java.util.Set; +import java.util.zip.GZIPInputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; +import org.apache.commons.compress.compressors.xz.XZCompressorInputStream; +import org.springframework.stereotype.Component; + +/** Inspects untrusted Terraform archives without extracting or executing their content. */ +@Component +final class TerraformArchiveInspector { + private static final long MAX_UPLOAD = 1024L * 1024 * 1024; + private static final long MAX_EXPANDED = 2L * 1024 * 1024 * 1024; + private static final long RATIO_FLOOR = 1024L * 1024; + private static final long MAX_COMPRESSION_RATIO = 200; + private static final int MAX_ENTRIES = 20_000; + private static final long MAX_INSPECTION_NANOS = Duration.ofMinutes(2).toNanos(); + + Path bufferAndInspect(InputStream body, String filename, boolean module, String providerName) { + Path file = null; + try { + file = Files.createTempFile("kkrepo-terraform-", suffix(filename)); + try (var out = Files.newOutputStream(file)) { + byte[] buffer = new byte[64 * 1024]; + long total = 0; + for (int read; (read = body.read(buffer)) >= 0;) { + total += read; + if (total > MAX_UPLOAD) throw bad("Terraform archive exceeds the upload limit"); + out.write(buffer, 0, read); + } + } + boolean valid = filename.toLowerCase(Locale.ROOT).endsWith(".zip") + ? inspectZip(file, module, providerName) + : inspectTar(file, filename, module, providerName); + if (!valid) { + throw bad(module + ? "Terraform module archive must contain at least one .tf file" + : "Terraform provider archive does not contain the expected provider binary"); + } + return file; + } catch (IOException | RuntimeException e) { + if (file != null) try { Files.deleteIfExists(file); } catch (IOException ignored) {} + if (e instanceof RuntimeException runtime) throw runtime; + throw bad("Unable to inspect Terraform archive", e); + } + } + + private boolean inspectZip(Path file, boolean module, String providerName) throws IOException { + try (ZipArchiveInputStream in = new ZipArchiveInputStream( + new BufferedInputStream(Files.newInputStream(file)), "UTF-8", true, true)) { + ZipArchiveEntry entry; + State state = new State(Files.size(file)); + while ((entry = in.getNextEntry()) != null) { + validateEntry(entry.getName(), entry.isUnixSymlink(), entry.isDirectory(), state); + if (!entry.isDirectory()) { + state.match |= matches(entry.getName(), module, providerName); + drain(in, state); + } + } + return state.match; + } + } + + private boolean inspectTar(Path file, String filename, boolean module, String providerName) throws IOException { + try (InputStream raw = Files.newInputStream(file); + InputStream compressed = decompressor(raw, filename); + TarArchiveInputStream in = new TarArchiveInputStream(compressed)) { + TarArchiveEntry entry; + State state = new State(Files.size(file)); + while ((entry = in.getNextEntry()) != null) { + validateEntry(entry.getName(), entry.isSymbolicLink() || entry.isLink(), entry.isDirectory(), state); + if (entry.isCharacterDevice() || entry.isBlockDevice() || entry.isFIFO()) { + throw bad("Terraform archive contains a device or FIFO entry"); + } + if (!entry.isDirectory()) { + state.match |= matches(entry.getName(), module, providerName); + drain(in, state); + } + } + return state.match; + } + } + + private static InputStream decompressor(InputStream raw, String filename) throws IOException { + String lower = filename == null ? "" : filename.toLowerCase(Locale.ROOT); + if (lower.endsWith(".tar.gz") || lower.endsWith(".tgz")) { + return new GZIPInputStream(raw); + } + if (lower.endsWith(".tar.xz") || lower.endsWith(".txz") || lower.endsWith(".xz")) { + return new XZCompressorInputStream(raw, true); + } + if (lower.endsWith(".tar.bz2") || lower.endsWith(".tbz2")) { + return new BZip2CompressorInputStream(raw, true); + } + throw bad("Unsupported Terraform module archive compression"); + } + + private static void validateEntry(String rawName, boolean link, boolean directory, State state) { + if (++state.entries > MAX_ENTRIES) throw bad("Terraform archive contains too many entries"); + String name = rawName == null ? "" : rawName.replace('\\', '/'); + if (name.isBlank() || name.startsWith("/") || name.startsWith("//") + || name.matches("^[A-Za-z]:.*") || name.indexOf('\0') >= 0 || link) { + throw bad("Terraform archive contains an unsafe entry"); + } + for (String segment : name.split("/", -1)) { + if (segment.isEmpty() && !directory || ".".equals(segment) || "..".equals(segment)) { + throw bad("Terraform archive contains path traversal"); + } + } + if (!state.names.add(name)) { + throw bad("Terraform archive contains duplicate entries"); + } + } + + private static boolean matches(String name, boolean module, String providerName) { + String leaf = name.replace('\\', '/'); + leaf = leaf.substring(leaf.lastIndexOf('/') + 1); + return module ? leaf.endsWith(".tf") + : leaf.startsWith("terraform-provider-" + providerName); + } + + private static void drain(InputStream in, State state) throws IOException { + byte[] buffer = new byte[32 * 1024]; + for (int read; (read = in.read(buffer)) >= 0;) { + state.expanded += read; + if (state.expanded > MAX_EXPANDED) throw bad("Terraform archive expands beyond the safe limit"); + if (state.expanded > RATIO_FLOOR + && state.expanded > Math.max(1L, state.compressedSize) * MAX_COMPRESSION_RATIO) { + throw bad("Terraform archive compression ratio exceeds the safe limit"); + } + if (System.nanoTime() - state.startedNanos > MAX_INSPECTION_NANOS) { + throw bad("Terraform archive inspection exceeded the time limit"); + } + } + } + + private static String suffix(String filename) { + String lower = filename == null ? "" : filename.toLowerCase(Locale.ROOT); + if (lower.endsWith(".tar.gz") || lower.endsWith(".tgz")) return ".tar.gz"; + if (lower.endsWith(".tar.xz") || lower.endsWith(".txz") || lower.endsWith(".xz")) return ".tar.xz"; + if (lower.endsWith(".tar.bz2") || lower.endsWith(".tbz2")) return ".tar.bz2"; + if (lower.endsWith(".zip")) return ".zip"; + throw bad("Terraform module archive must be .zip, .tar.gz, .tgz, .txz, .xz, .tar.xz, .tar.bz2, or .tbz2"); + } + + private static MavenExceptions.BadRequestException bad(String message) { + return new MavenExceptions.BadRequestException(message); + } + + private static MavenExceptions.BadRequestException bad(String message, Throwable cause) { + return new MavenExceptions.BadRequestException(message, cause); + } + + private static final class State { + final long compressedSize; + final long startedNanos = System.nanoTime(); + final Set names = new HashSet<>(); + int entries; + long expanded; + boolean match; + + State(long compressedSize) { + this.compressedSize = compressedSize; + } + } +} diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformAssetSupport.java b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformAssetSupport.java new file mode 100644 index 00000000..95c2f0bf --- /dev/null +++ b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformAssetSupport.java @@ -0,0 +1,82 @@ +package com.github.klboke.kkrepo.server.terraform; + +import com.github.klboke.kkrepo.persistence.jdbc.api.AssetDao; +import com.github.klboke.kkrepo.persistence.jdbc.api.model.AssetBlobRecord; +import com.github.klboke.kkrepo.persistence.jdbc.api.model.AssetRecord; +import com.github.klboke.kkrepo.server.blob.BlobReferenceCodec; +import com.github.klboke.kkrepo.server.maven.BlobStorageRegistry; +import com.github.klboke.kkrepo.server.maven.MavenExceptions; +import com.github.klboke.kkrepo.server.maven.MavenResponse; +import com.github.klboke.kkrepo.server.maven.RepositoryRuntime; +import com.github.klboke.kkrepo.server.raw.RawHostedService; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Map; +import java.util.Optional; +import org.springframework.stereotype.Component; + +@Component +final class TerraformAssetSupport { + private final AssetDao assets; + private final BlobStorageRegistry storages; + private final RawHostedService hosted; + + TerraformAssetSupport(AssetDao assets, BlobStorageRegistry storages, RawHostedService hosted) { + this.assets = assets; + this.storages = storages; + this.hosted = hosted; + } + + void store(RepositoryRuntime runtime, String path, InputStream body, String contentType, + Map attributes, String actor, String ip) { + hosted.putInternal(runtime, path, body, contentType, attributes, actor, ip); + } + + void storeBytes(RepositoryRuntime runtime, String path, byte[] body, String contentType, + Map attributes) { + store(runtime, path, new ByteArrayInputStream(body), contentType, attributes, "terraform", null); + } + + Optional find(RepositoryRuntime runtime, String path) { + return assets.findAssetByPath(runtime.id(), path); + } + + java.util.List list(RepositoryRuntime runtime, String prefix) { + return assets.listAssetsByPrefix(runtime.id(), prefix); + } + + AssetBlobRecord blob(AssetRecord asset) { + return asset.assetBlobId() == null ? null : assets.findBlobById(asset.assetBlobId()).orElse(null); + } + + byte[] bytes(RepositoryRuntime runtime, String path) { + AssetRecord asset = find(runtime, path).orElseThrow(() -> notFound(path)); + AssetBlobRecord blob = blob(asset); + if (blob == null) throw notFound(path); + try (InputStream in = storages.forBlobStoreId(blob.blobStoreId()).get( + BlobReferenceCodec.reference(blob.blobRef(), blob.objectKey(), blob.sha256(), blob.size())) + .orElseThrow(() -> notFound(path))) { + return in.readAllBytes(); + } catch (IOException e) { + throw new MavenExceptions.BadUpstreamException("Failed reading Terraform asset " + path, e); + } + } + + MavenResponse serve(RepositoryRuntime runtime, String path, boolean headOnly) { + AssetRecord asset = find(runtime, path).orElseThrow(() -> notFound(path)); + AssetBlobRecord blob = blob(asset); + if (blob == null) throw notFound(path); + var ref = BlobReferenceCodec.reference(blob.blobRef(), blob.objectKey(), blob.sha256(), blob.size()); + var storage = storages.forBlobStoreId(blob.blobStoreId()); + if (storage.stat(ref).isEmpty()) throw notFound(path); + return headOnly + ? MavenResponse.noBody(200, blob.size(), asset.contentType(), blob.sha1(), asset.lastUpdatedAt()) + : MavenResponse.ok(() -> storage.get(ref).orElseThrow(() -> notFound(path)), + blob.size(), asset.contentType(), blob.sha1(), asset.lastUpdatedAt()); + } + + private static MavenExceptions.MavenNotFoundException notFound(String path) { + return new MavenExceptions.MavenNotFoundException(path); + } +} diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformPublishLeaseManager.java b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformPublishLeaseManager.java new file mode 100644 index 00000000..8c0dd176 --- /dev/null +++ b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformPublishLeaseManager.java @@ -0,0 +1,63 @@ +package com.github.klboke.kkrepo.server.terraform; + +import com.github.klboke.kkrepo.persistence.jdbc.api.TerraformRegistryDao; +import com.github.klboke.kkrepo.server.maven.MavenExceptions; +import java.time.Duration; +import java.time.Instant; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.LockSupport; +import org.springframework.stereotype.Component; + +/** Blocking, database-backed lease acquisition used to serialize publication across replicas. */ +@Component +final class TerraformPublishLeaseManager { + private static final long RETRY_NANOS = Duration.ofMillis(25).toNanos(); + + private final TerraformRegistryDao registry; + + TerraformPublishLeaseManager(TerraformRegistryDao registry) { + this.registry = registry; + } + + Lease acquire(String key, Duration ttl, Duration wait) { + String owner = UUID.randomUUID().toString(); + long deadline = System.nanoTime() + wait.toNanos(); + do { + if (registry.tryAcquirePublishLease(key, owner, Instant.now().plus(ttl))) { + return new Lease(registry, key, owner); + } + if (Thread.currentThread().isInterrupted()) { + Thread.currentThread().interrupt(); + throw unavailable(key); + } + LockSupport.parkNanos(RETRY_NANOS); + } while (System.nanoTime() < deadline); + throw unavailable(key); + } + + private static MavenExceptions.WritePolicyDenied unavailable(String key) { + return new MavenExceptions.WritePolicyDenied( + "Terraform publication lease is busy; retry the request: " + key); + } + + static final class Lease implements AutoCloseable { + private final TerraformRegistryDao registry; + private final String key; + private final String owner; + private final AtomicBoolean closed = new AtomicBoolean(); + + private Lease(TerraformRegistryDao registry, String key, String owner) { + this.registry = registry; + this.key = key; + this.owner = owner; + } + + @Override + public void close() { + if (closed.compareAndSet(false, true)) { + registry.releasePublishLease(key, owner); + } + } + } +} diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformRepositoryDataMigrationWriter.java b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformRepositoryDataMigrationWriter.java new file mode 100644 index 00000000..6ca54e77 --- /dev/null +++ b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformRepositoryDataMigrationWriter.java @@ -0,0 +1,171 @@ +package com.github.klboke.kkrepo.server.terraform; + +import com.github.klboke.kkrepo.core.RepositoryFormat; +import com.github.klboke.kkrepo.persistence.jdbc.api.TerraformRegistryDao; +import com.github.klboke.kkrepo.persistence.jdbc.api.model.AssetBlobRecord; +import com.github.klboke.kkrepo.persistence.jdbc.api.model.AssetRecord; +import com.github.klboke.kkrepo.persistence.jdbc.api.model.RepositoryDataMigrationAssetRecord; +import com.github.klboke.kkrepo.persistence.jdbc.api.model.RepositoryRecord; +import com.github.klboke.kkrepo.protocol.terraform.TerraformPath; +import com.github.klboke.kkrepo.protocol.terraform.TerraformPathParser; +import com.github.klboke.kkrepo.server.maven.RepositoryRuntime; +import com.github.klboke.kkrepo.server.maven.RepositoryRuntimeRegistry; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.springframework.stereotype.Component; + +/** Replays Nexus Terraform archives through the normal validation and atomic publication path. */ +@Component +public class TerraformRepositoryDataMigrationWriter { + private static final Pattern NEXUS_PROVIDER_ARCHIVE = Pattern.compile( + "^v1/providers/([^/]+)/([^/]+)/([^/]+)/download/([^/]+)/([^/]+)/([^/]+\\.zip)$", + Pattern.CASE_INSENSITIVE); + + private final TerraformService service; + private final TerraformAssetSupport assets; + private final TerraformRegistryDao registry; + private final RepositoryRuntimeRegistry runtimes; + private final TerraformPathParser paths = new TerraformPathParser(); + + TerraformRepositoryDataMigrationWriter( + TerraformService service, + TerraformAssetSupport assets, + TerraformRegistryDao registry, + RepositoryRuntimeRegistry runtimes) { + this.service = service; + this.assets = assets; + this.registry = registry; + this.runtimes = runtimes; + } + + public MigratedAsset write( + RepositoryRecord repository, + RepositoryDataMigrationAssetRecord source, + InputStream body, + String responseContentType, + boolean validateSize) { + if (repository.format() != RepositoryFormat.TERRAFORM) { + throw new IllegalArgumentException("Terraform migration writer requires a Terraform repository"); + } + RepositoryRuntime runtime = runtimes.resolveById(repository.id()) + .orElseThrow(() -> new IllegalArgumentException( + "Terraform migration target repository is unavailable: " + repository.name())); + SourceTarget target = target(source.sourcePath()); + Optional existing = assets.find(runtime, target.assetPath()); + if (existing.isPresent() && target.module()) { + return verifiedExisting(source, existing.get(), validateSize); + } + if (existing.isPresent() && !target.module() + && registry.listProviderPlatforms( + runtime.id(), target.path().namespace(), target.path().name(), target.path().version()).stream() + .anyMatch(platform -> platform.os().equals(target.path().os()) + && platform.arch().equals(target.path().arch()) + && platform.assetPath().equals(target.assetPath()))) { + return verifiedExisting(source, existing.get(), validateSize); + } + + Path buffered = null; + try { + buffered = Files.createTempFile("kkrepo-terraform-migration-", target.module() ? ".archive" : ".zip"); + long size = Files.copy(body, buffered, java.nio.file.StandardCopyOption.REPLACE_EXISTING); + if (validateSize && source.size() != null && source.size() >= 0 && source.size() != size) { + throw new IllegalStateException("Terraform migration size mismatch for " + source.sourcePath() + + ": expected " + source.size() + ", actual " + size); + } + try (InputStream replay = Files.newInputStream(buffered)) { + service.put( + runtime, + target.path(), + replay, + firstNonBlank(responseContentType, source.contentType()), + target.module() ? null : "attachment; filename=\"" + target.filename() + "\"", + firstNonBlank(source.sourceCreatedBy(), "nexus-migration"), + source.sourceCreatedByIp()); + } + AssetRecord stored = assets.find(runtime, target.assetPath()) + .orElseThrow(() -> new IllegalStateException( + "Terraform migration did not publish " + target.assetPath())); + return verifiedExisting(source, stored, validateSize); + } catch (IOException e) { + throw new IllegalStateException("Failed buffering Terraform migration asset " + source.sourcePath(), e); + } finally { + if (buffered != null) { + try { + Files.deleteIfExists(buffered); + } catch (IOException ignored) { + } + } + } + } + + public static boolean isMigratableTerraformPath(String rawPath) { + try { + String path = normalize(rawPath); + TerraformPath parsed = new TerraformPathParser().parse(path); + return parsed.kind() == TerraformPath.Kind.MODULE_ARCHIVE + || NEXUS_PROVIDER_ARCHIVE.matcher(path).matches(); + } catch (IllegalArgumentException e) { + return false; + } + } + + private MigratedAsset verifiedExisting( + RepositoryDataMigrationAssetRecord source, AssetRecord asset, boolean validateSize) { + AssetBlobRecord blob = assets.blob(asset); + if (blob == null) { + throw new IllegalStateException("Migrated Terraform asset has no blob: " + asset.path()); + } + if (validateSize && source.size() != null && source.size() >= 0 && source.size() != blob.size()) { + throw new IllegalStateException("Terraform migration size mismatch for " + source.sourcePath() + + ": expected " + source.size() + ", actual " + blob.size()); + } + return new MigratedAsset(asset.componentId(), asset.id(), blob.id(), blob.objectKey()); + } + + private SourceTarget target(String rawPath) { + String normalized = normalize(rawPath); + TerraformPath module = paths.parse(normalized); + if (module.kind() == TerraformPath.Kind.MODULE_ARCHIVE) { + return new SourceTarget(module, normalized, module.filename(), true); + } + Matcher provider = NEXUS_PROVIDER_ARCHIVE.matcher(normalized); + if (!provider.matches()) { + throw new IllegalArgumentException("Unsupported Nexus Terraform migration asset: " + rawPath); + } + String uploadPath = "v1/providers/" + provider.group(1) + "/" + provider.group(2) + "/" + + provider.group(3) + "/download/" + provider.group(4) + "/" + provider.group(5); + TerraformPath parsed = paths.parse(uploadPath); + if (parsed.kind() != TerraformPath.Kind.PROVIDER_DOWNLOAD) { + throw new IllegalArgumentException("Invalid Nexus Terraform provider asset: " + rawPath); + } + TerraformPathParser.requireFilename(provider.group(6)); + String targetPath = "v1/providers/" + parsed.namespace() + "/" + parsed.name() + "/" + + parsed.version() + "/package/" + parsed.os() + "/" + provider.group(6); + return new SourceTarget(parsed, targetPath, provider.group(6), false); + } + + private static String normalize(String path) { + String value = path == null ? "" : path.trim(); + while (value.startsWith("/")) value = value.substring(1); + while (value.endsWith("/")) value = value.substring(0, value.length() - 1); + return value; + } + + private static String firstNonBlank(String first, String second) { + if (first != null && !first.isBlank()) return first; + return second == null || second.isBlank() ? "application/octet-stream" : second; + } + + public record MigratedAsset( + Long componentId, long assetId, long assetBlobId, String assetBlobObjectKey) { + } + + private record SourceTarget( + TerraformPath path, String assetPath, String filename, boolean module) { + } +} diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java new file mode 100644 index 00000000..c018586b --- /dev/null +++ b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java @@ -0,0 +1,696 @@ +package com.github.klboke.kkrepo.server.terraform; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.klboke.kkrepo.persistence.jdbc.api.TerraformRegistryDao; +import com.github.klboke.kkrepo.persistence.jdbc.api.model.AssetBlobRecord; +import com.github.klboke.kkrepo.persistence.jdbc.api.model.AssetRecord; +import com.github.klboke.kkrepo.protocol.terraform.TerraformPath; +import com.github.klboke.kkrepo.protocol.terraform.TerraformPathParser; +import com.github.klboke.kkrepo.protocol.terraform.TerraformVersions; +import com.github.klboke.kkrepo.server.maven.HttpRemoteFetcher; +import com.github.klboke.kkrepo.server.maven.MavenExceptions; +import com.github.klboke.kkrepo.server.maven.MavenResponse; +import com.github.klboke.kkrepo.server.maven.RepositoryRuntime; +import com.github.klboke.kkrepo.server.maven.RepositoryRuntimeRegistry; +import com.github.klboke.kkrepo.server.raw.RawProxyService; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Service; + +/** HashiCorp Registry Protocol implementation shared by hosted, proxy, and group recipes. */ +@Service +public class TerraformService { + private static final String JSON = MediaType.APPLICATION_JSON_VALUE; + private static final String OCTET = MediaType.APPLICATION_OCTET_STREAM_VALUE; + private static final String PROTOCOLS = "5.0"; + private static final Pattern CONTENT_DISPOSITION_FILENAME = Pattern.compile( + "(?i)(?:^|;)\\s*filename\\s*=\\s*(?:\"([^\"]+)\"|([^;]+))\\s*(?:;|$)"); + + private final ObjectMapper mapper; + private final TerraformAssetSupport assets; + private final TerraformArchiveInspector inspector; + private final TerraformSigningService signing; + private final TerraformSignatureVerifier signatureVerifier; + private final TerraformRegistryDao registry; + private final TerraformPublishLeaseManager leases; + private final RepositoryRuntimeRegistry runtimes; + private final RawProxyService proxy; + private final HttpRemoteFetcher fetcher; + private final TerraformPathParser paths = new TerraformPathParser(); + + public TerraformService( + ObjectMapper mapper, + TerraformAssetSupport assets, + TerraformArchiveInspector inspector, + TerraformSigningService signing, + TerraformSignatureVerifier signatureVerifier, + TerraformRegistryDao registry, + TerraformPublishLeaseManager leases, + RepositoryRuntimeRegistry runtimes, + RawProxyService proxy, + HttpRemoteFetcher fetcher) { + this.mapper = mapper; + this.assets = assets; + this.inspector = inspector; + this.signing = signing; + this.signatureVerifier = signatureVerifier; + this.registry = registry; + this.leases = leases; + this.runtimes = runtimes; + this.proxy = proxy; + this.fetcher = fetcher; + } + + public MavenResponse get( + RepositoryRuntime runtime, TerraformPath path, String repositoryBaseUrl, boolean headOnly) { + return get(runtime, path, repositoryBaseUrl, null, headOnly); + } + + public MavenResponse get( + RepositoryRuntime runtime, + TerraformPath path, + String repositoryBaseUrl, + String urlTokenSegment, + boolean headOnly) { + return get(runtime, path, new RequestUrls(repositoryBaseUrl, urlTokenSegment), headOnly); + } + + private MavenResponse get( + RepositoryRuntime runtime, TerraformPath path, RequestUrls urls, boolean headOnly) { + if (path.kind() == TerraformPath.Kind.UNKNOWN) throw notFound(path.rawPath()); + return switch (runtime.type()) { + case HOSTED -> hostedGet(runtime, path, urls, headOnly); + case PROXY -> proxyGet(runtime, path, urls, headOnly); + case GROUP -> groupGet(runtime, path, urls, headOnly); + }; + } + + public MavenResponse put( + RepositoryRuntime runtime, + TerraformPath path, + InputStream body, + String contentType, + String contentDisposition, + String actor, + String ip) { + if (!runtime.isHosted()) throw new MavenExceptions.MethodNotAllowed("Terraform group/proxy repositories are read-only"); + return switch (path.kind()) { + case MODULE_ARCHIVE -> putModule(runtime, path, body, contentType, actor, ip); + case PROVIDER_DOWNLOAD -> putProvider(runtime, path, body, contentType, contentDisposition, actor, ip); + default -> throw new MavenExceptions.MethodNotAllowed("Unsupported Terraform PUT path: " + path.rawPath()); + }; + } + + private MavenResponse hostedGet( + RepositoryRuntime runtime, TerraformPath path, RequestUrls urls, boolean headOnly) { + return switch (path.kind()) { + case MODULE_VERSIONS -> json(moduleVersions(runtime, path), headOnly); + case MODULE_DOWNLOAD -> moduleDownload(runtime, path, urls); + case MODULE_ARCHIVE -> assets.serve(runtime, path.rawPath(), headOnly); + case PROVIDER_VERSIONS -> json(providerVersions(runtime, path), headOnly); + case PROVIDER_DOWNLOAD -> providerDownload(runtime, path, urls, headOnly); + case PROVIDER_ARCHIVE -> servePublishedProviderArchive(runtime, path, headOnly); + case PROVIDER_SHA256SUMS, PROVIDER_SHA256SUMS_SIGNATURE -> + servePublishedProviderMetadata(runtime, path, headOnly); + default -> throw notFound(path.rawPath()); + }; + } + + private Map moduleVersions(RepositoryRuntime runtime, TerraformPath request) { + String prefix = "v1/modules/" + request.namespace() + "/" + request.name() + "/" + request.system() + "/"; + Set versions = new LinkedHashSet<>(); + for (AssetRecord asset : assets.list(runtime, prefix)) { + TerraformPath parsed = paths.parse(asset.path()); + if (parsed.kind() == TerraformPath.Kind.MODULE_ARCHIVE) versions.add(parsed.version()); + } + List> rows = TerraformVersions.descending(versions).stream() + .map(version -> Map.of("version", version)).toList(); + return Map.of("modules", List.of(Map.of( + "source", request.namespace() + "/" + request.name() + "/" + request.system(), + "versions", rows))); + } + + private MavenResponse moduleDownload(RepositoryRuntime runtime, TerraformPath request, RequestUrls urls) { + String prefix = "v1/modules/" + request.namespace() + "/" + request.name() + "/" + + request.system() + "/" + request.version() + "/"; + AssetRecord archive = assets.list(runtime, prefix).stream() + .filter(asset -> paths.parse(asset.path()).kind() == TerraformPath.Kind.MODULE_ARCHIVE) + .min(Comparator.comparing(AssetRecord::path)) + .orElseThrow(() -> notFound(prefix)); + return MavenResponse.noBody(204).withHeader("X-Terraform-Get", publicUrl(urls, archive.path())); + } + + private Map providerVersions(RepositoryRuntime runtime, TerraformPath request) { + String prefix = "v1/providers/" + request.namespace() + "/" + request.name() + "/"; + Set versions = new LinkedHashSet<>(); + for (AssetRecord asset : assets.list(runtime, prefix)) { + TerraformPath parsed = paths.parse(asset.path()); + if (parsed.kind() == TerraformPath.Kind.PROVIDER_ARCHIVE + && registry.findProviderState(runtime.id(), request.namespace(), request.name(), parsed.version()).isPresent()) { + versions.add(parsed.version()); + } + } + List> values = new ArrayList<>(); + for (String version : TerraformVersions.descending(versions)) { + List> platforms = registry.listProviderPlatforms( + runtime.id(), request.namespace(), request.name(), version).stream() + .map(row -> Map.of("os", row.os(), "arch", row.arch())) + .toList(); + values.add(Map.of("version", version, "protocols", List.of(PROTOCOLS), "platforms", platforms)); + } + return Map.of("versions", values); + } + + private MavenResponse providerDownload( + RepositoryRuntime runtime, TerraformPath request, RequestUrls urls, boolean headOnly) { + TerraformRegistryDao.ProviderState state = registry.findProviderState( + runtime.id(), request.namespace(), request.name(), request.version()) + .orElseThrow(() -> notFound(request.rawPath())); + TerraformRegistryDao.ProviderPlatform platform = registry.listProviderPlatforms( + runtime.id(), request.namespace(), request.name(), request.version()).stream() + .filter(row -> row.os().equals(request.os()) && row.arch().equals(request.arch())) + .findFirst().orElseThrow(() -> notFound(request.rawPath())); + TerraformRegistryDao.SigningKey key = registry.findSigningKey(runtime.id(), state.signingKeyRevision()) + .orElseThrow(() -> new IllegalStateException("Terraform signing key revision is missing")); + Map body = new LinkedHashMap<>(); + body.put("protocols", protocolList(platform.protocols())); + body.put("os", platform.os()); + body.put("arch", platform.arch()); + body.put("filename", platform.filename()); + body.put("download_url", publicUrl(urls, platform.assetPath())); + body.put("shasums_url", publicUrl(urls, state.shasumsPath())); + body.put("shasums_signature_url", publicUrl(urls, state.signaturePath())); + body.put("shasum", platform.sha256()); + body.put("signing_keys", Map.of("gpg_public_keys", List.of(Map.of( + "key_id", key.keyId(), "ascii_armor", key.publicKey(), "trust_signature", "")))); + return json(body, headOnly); + } + + private MavenResponse servePublishedProviderArchive( + RepositoryRuntime runtime, TerraformPath path, boolean headOnly) { + boolean published = registry.listProviderPlatforms( + runtime.id(), path.namespace(), path.name(), path.version()).stream() + .anyMatch(row -> row.assetPath().equals(path.rawPath())); + if (!published) throw notFound(path.rawPath()); + return assets.serve(runtime, path.rawPath(), headOnly); + } + + private MavenResponse servePublishedProviderMetadata( + RepositoryRuntime runtime, TerraformPath path, boolean headOnly) { + TerraformRegistryDao.ProviderState state = registry.findProviderState( + runtime.id(), path.namespace(), path.name(), path.version()) + .orElseThrow(() -> notFound(path.rawPath())); + String published = path.kind() == TerraformPath.Kind.PROVIDER_SHA256SUMS + ? state.shasumsPath() : state.signaturePath(); + if (!published.equals(path.rawPath())) throw notFound(path.rawPath()); + return assets.serve(runtime, path.rawPath(), headOnly); + } + + private MavenResponse putModule( + RepositoryRuntime runtime, TerraformPath path, InputStream body, String contentType, + String actor, String ip) { + enforceWrite(runtime, path.rawPath()); + Path buffered = inspector.bufferAndInspect(body, path.filename(), true, null); + try (InputStream in = Files.newInputStream(buffered)) { + assets.store(runtime, path.rawPath(), in, contentType == null ? OCTET : contentType, + Map.of( + "terraformKind", "module-archive", + "namespace", path.namespace(), "name", path.name(), "system", path.system(), + "version", path.version(), "sha256Validated", true), actor, ip); + return MavenResponse.created(); + } catch (IOException e) { + throw new IllegalStateException("Failed storing Terraform module archive", e); + } finally { + try { Files.deleteIfExists(buffered); } catch (IOException ignored) {} + } + } + + private MavenResponse putProvider( + RepositoryRuntime runtime, TerraformPath path, InputStream body, String contentType, + String contentDisposition, String actor, String ip) { + String filename = filename(contentDisposition); + String expectedPrefix = "terraform-provider-" + path.name() + "_" + path.version() + + "_" + path.os() + "_" + path.arch(); + if (!filename.startsWith(expectedPrefix) || !filename.toLowerCase(Locale.ROOT).endsWith(".zip")) { + throw new MavenExceptions.BadRequestException( + "Provider filename must match " + expectedPrefix + "*.zip"); + } + String leaseKey = runtime.id() + ":" + path.namespace() + ":" + path.name() + ":" + path.version(); + try (TerraformPublishLeaseManager.Lease lease = leases.acquire( + leaseKey, java.time.Duration.ofMinutes(5), java.time.Duration.ofSeconds(30))) { + List current = registry.listProviderPlatforms( + runtime.id(), path.namespace(), path.name(), path.version()); + boolean exists = current.stream().anyMatch(row -> row.os().equals(path.os()) && row.arch().equals(path.arch())); + if (exists) throw new MavenExceptions.WritePolicyDenied("Terraform provider platform already exists"); + long revision = registry.findProviderState(runtime.id(), path.namespace(), path.name(), path.version()) + .map(state -> state.revision() + 1).orElse(1L); + String assetPath = "v1/providers/" + path.namespace() + "/" + path.name() + "/" + path.version() + + "/package/" + path.os() + "/" + filename; + if (assets.find(runtime, assetPath).isEmpty()) { + Path buffered = inspector.bufferAndInspect(body, filename, false, path.name()); + try (InputStream in = Files.newInputStream(buffered)) { + assets.store(runtime, assetPath, in, contentType == null ? OCTET : contentType, + Map.of( + "terraformKind", "provider-archive", "namespace", path.namespace(), "type", path.name(), + "version", path.version(), "os", path.os(), "arch", path.arch(), + "protocols", List.of(PROTOCOLS), "publishState", "STAGING", "revision", revision), actor, ip); + } finally { + try { Files.deleteIfExists(buffered); } catch (IOException ignored) {} + } + } + AssetRecord stored = assets.find(runtime, assetPath).orElseThrow(() -> notFound(assetPath)); + AssetBlobRecord blob = assets.blob(stored); + if (blob == null || blob.sha256() == null) throw new IllegalStateException("Provider archive digest is missing"); + TerraformRegistryDao.ProviderPlatform published = new TerraformRegistryDao.ProviderPlatform( + runtime.id(), path.namespace(), path.name(), path.version(), path.os(), path.arch(), filename, + assetPath, blob.sha256(), PROTOCOLS, revision, Instant.now()); + List next = new ArrayList<>(current); + next.add(published); + next.sort(Comparator.comparing(TerraformRegistryDao.ProviderPlatform::filename)); + byte[] shasums = next.stream().map(row -> row.sha256() + " " + row.filename() + "\n") + .collect(java.util.stream.Collectors.joining()).getBytes(StandardCharsets.UTF_8); + TerraformSigningService.SigningMaterial key = signing.active(runtime); + byte[] signature = signing.sign(shasums, key); + String fileBase = "terraform-provider-" + path.name() + "_" + path.version() + "_SHA256SUMS"; + String metadataBase = "v1/providers/" + path.namespace() + "/" + path.name() + "/" + path.version() + + "/metadata-r" + revision + "/"; + String sumsPath = metadataBase + fileBase; + String signaturePath = sumsPath + ".sig"; + assets.storeBytes(runtime, sumsPath, shasums, MediaType.TEXT_PLAIN_VALUE, + Map.of("terraformKind", "provider-shasums", "revision", revision)); + assets.storeBytes(runtime, signaturePath, signature, OCTET, + Map.of("terraformKind", "provider-signature", "revision", revision, "keyId", key.keyId())); + registry.publishProvider(published, new TerraformRegistryDao.ProviderState( + runtime.id(), path.namespace(), path.name(), path.version(), revision, + sumsPath, signaturePath, key.revision(), Instant.now())); + return MavenResponse.created(); + } catch (IOException e) { + throw new IllegalStateException("Failed storing Terraform provider archive", e); + } + } + + private MavenResponse proxyGet( + RepositoryRuntime runtime, TerraformPath path, RequestUrls urls, boolean headOnly) { + return switch (path.kind()) { + case MODULE_VERSIONS, PROVIDER_VERSIONS -> proxyMetadata(runtime, path, headOnly); + case MODULE_DOWNLOAD -> proxyModuleDownload(runtime, path, urls); + case PROVIDER_DOWNLOAD -> proxyProviderDownload(runtime, path, urls, headOnly); + case MODULE_ARCHIVE, PROVIDER_ARCHIVE, PROVIDER_SHA256SUMS, PROVIDER_SHA256SUMS_SIGNATURE -> + proxyRoute(runtime, path.rawPath(), headOnly); + default -> throw notFound(path.rawPath()); + }; + } + + private MavenResponse proxyMetadata(RepositoryRuntime runtime, TerraformPath path, boolean headOnly) { + String remote = remoteUrl(runtime, path); + String cachePath = ".terraform/upstream/" + sha256(remote) + ".json"; + MavenResponse cached = proxy.getAssetFromUrl(runtime, cachePath, remote, false); + byte[] bytes = responseBytes(cached); + return bytes(bytes, JSON, headOnly); + } + + private MavenResponse proxyModuleDownload(RepositoryRuntime runtime, TerraformPath path, RequestUrls urls) { + String remote = remoteUrl(runtime, path); + try { + HttpRemoteFetcher.Request request = HttpRemoteFetcher.Request.get(remote) + .withTimeoutProfile(HttpRemoteFetcher.TimeoutProfile.METADATA).withRepository(runtime); + return fetcher.fetchWithBodyRetry(request, path.rawPath(), result -> { + if (result.status() == 404) throw notFound(path.rawPath()); + if (result.status() < 200 || result.status() >= 300) { + throw new MavenExceptions.BadUpstreamException("Terraform upstream returned " + result.status()); + } + String upstream = result.header("X-Terraform-Get"); + if (upstream == null || upstream.isBlank()) { + throw new MavenExceptions.BadUpstreamException("Terraform upstream omitted X-Terraform-Get"); + } + String absolute = URI.create(remote).resolve(upstream).toString(); + String filename = safeRemoteFilename(absolute, + path.name() + "_" + path.version() + ".zip"); + String local = "v1/modules/" + path.namespace() + "/" + path.name() + "/" + path.system() + + "/" + path.version() + "/" + filename; + storeRoute(runtime, local, absolute, null); + return MavenResponse.noBody(204).withHeader("X-Terraform-Get", publicUrl(urls, local)); + }); + } catch (IOException e) { + throw new MavenExceptions.BadUpstreamException("Failed reading Terraform module upstream", e); + } + } + + @SuppressWarnings("unchecked") + private MavenResponse proxyProviderDownload( + RepositoryRuntime runtime, TerraformPath path, RequestUrls urls, boolean headOnly) { + String remote = remoteUrl(runtime, path); + String cachePath = ".terraform/upstream/" + sha256(remote) + ".json"; + Map body = readJson(responseBytes(proxy.getAssetFromUrl(runtime, cachePath, remote, false))); + String filename = string(body.get("filename")); + TerraformPathParser.requireFilename(filename); + String download = absolute(remote, string(body.get("download_url"))); + String sums = absolute(remote, string(body.get("shasums_url"))); + String signature = absolute(remote, string(body.get("shasums_signature_url"))); + String localDownload = "v1/providers/" + path.namespace() + "/" + path.name() + "/" + path.version() + + "/package/" + path.os() + "/" + filename; + String sumsFile = safeRemoteFilename(sums, + "terraform-provider-" + path.name() + "_" + path.version() + "_SHA256SUMS"); + String localSums = "v1/providers/" + path.namespace() + "/" + path.name() + "/" + path.version() + + "/metadata-proxy/" + sumsFile; + String localSignature = localSums + ".sig"; + String expected = string(body.get("shasum")); + byte[] shasumsBytes = responseBytes(proxy.getAssetFromUrl(runtime, localSums, sums, false)); + byte[] signatureBytes = responseBytes(proxy.getAssetFromUrl(runtime, localSignature, signature, false)); + verifyChecksumEntry(shasumsBytes, expected, filename); + signatureVerifier.verify(shasumsBytes, signatureBytes, upstreamPublicKeys(body)); + storeRoute(runtime, localDownload, download, expected); + storeRoute(runtime, localSums, sums, null); + storeRoute(runtime, localSignature, signature, null); + Map rewritten = new LinkedHashMap<>(body); + rewritten.put("download_url", publicUrl(urls, localDownload)); + rewritten.put("shasums_url", publicUrl(urls, localSums)); + rewritten.put("shasums_signature_url", publicUrl(urls, localSignature)); + return json(rewritten, headOnly); + } + + @SuppressWarnings("unchecked") + private static List upstreamPublicKeys(Map body) { + Object signing = body.get("signing_keys"); + if (!(signing instanceof Map keys)) { + throw new MavenExceptions.BadUpstreamException("Terraform upstream omitted signing keys"); + } + Object gpg = keys.get("gpg_public_keys"); + if (!(gpg instanceof List values)) { + throw new MavenExceptions.BadUpstreamException("Terraform upstream omitted GPG signing keys"); + } + List armors = values.stream() + .filter(Map.class::isInstance) + .map(Map.class::cast) + .map(value -> string(value.get("ascii_armor"))) + .filter(value -> value != null && !value.isBlank()) + .toList(); + if (armors.isEmpty()) throw new MavenExceptions.BadUpstreamException("Terraform upstream omitted GPG signing keys"); + return armors; + } + + private static void verifyChecksumEntry(byte[] shasums, String expected, String filename) { + if (expected == null || expected.isBlank()) { + throw new MavenExceptions.BadUpstreamException("Terraform upstream omitted provider checksum"); + } + String expectedLine = expected.toLowerCase(Locale.ROOT) + " " + filename; + boolean present = StandardCharsets.UTF_8.decode(java.nio.ByteBuffer.wrap(shasums)).toString().lines() + .map(String::trim).anyMatch(line -> line.equalsIgnoreCase(expectedLine)); + if (!present) { + throw new MavenExceptions.BadUpstreamException("Terraform upstream checksum manifest does not contain " + filename); + } + } + + private MavenResponse proxyRoute(RepositoryRuntime runtime, String localPath, boolean headOnly) { + Map route = readJson(assets.bytes(runtime, routePath(localPath))); + String remote = string(route.get("remoteUrl")); + String expected = string(route.get("sha256")); + MavenResponse response = proxy.getAssetFromUrl(runtime, localPath, remote, headOnly); + if (expected != null && !expected.isBlank()) { + AssetRecord asset = assets.find(runtime, localPath).orElseThrow(() -> notFound(localPath)); + AssetBlobRecord blob = assets.blob(asset); + if (blob == null || !expected.equalsIgnoreCase(blob.sha256())) { + throw new MavenExceptions.BadUpstreamException("Terraform provider checksum mismatch for " + localPath); + } + } + return response; + } + + private MavenResponse groupGet( + RepositoryRuntime group, TerraformPath path, RequestUrls urls, boolean headOnly) { + if (headOnly && (path.kind() == TerraformPath.Kind.MODULE_ARCHIVE + || path.kind() == TerraformPath.Kind.PROVIDER_ARCHIVE + || path.kind() == TerraformPath.Kind.PROVIDER_SHA256SUMS + || path.kind() == TerraformPath.Kind.PROVIDER_SHA256SUMS_SIGNATURE)) { + // Nexus 3.92's Terraform group facet does not dispatch HEAD to member archive assets even + // though GET for the same URL succeeds. Keep this intentionally asymmetric behavior pinned + // by the black-box compatibility suite. + throw notFound(path.rawPath()); + } + if (path.kind() == TerraformPath.Kind.MODULE_VERSIONS + || path.kind() == TerraformPath.Kind.PROVIDER_VERSIONS) { + return mergeGroupVersions(group, path, urls, headOnly); + } + String bindingKey = "asset:" + path.rawPath(); + Optional existing = registry.findSourceBinding(group.id(), bindingKey); + if (existing.isPresent()) { + RepositoryRuntime member = runtimes.resolveById(existing.get().memberRepositoryId()).orElse(null); + if (member != null) { + try { return get(member, path, urls, headOnly); } + catch (MavenExceptions.MavenNotFoundException ignored) {} + } + } + for (RepositoryRuntime member : group.members()) { + try { + MavenResponse response = get(member, path, urls, headOnly); + return bindGroupResponse(group, member, path, response); + } catch (MavenExceptions.MavenNotFoundException | MavenExceptions.BadUpstreamException ignored) { + // Nexus groups probe the next member on a missing or unavailable member. + } + } + throw notFound(path.rawPath()); + } + + @SuppressWarnings("unchecked") + private MavenResponse mergeGroupVersions( + RepositoryRuntime group, TerraformPath path, RequestUrls urls, boolean headOnly) { + Map> versions = new LinkedHashMap<>(); + for (RepositoryRuntime member : group.members()) { + try { + Map body = readJson(responseBytes(get(member, path, urls, false))); + if (path.kind() == TerraformPath.Kind.MODULE_VERSIONS) { + List> modules = (List>) body.getOrDefault("modules", List.of()); + for (Map module : modules) { + for (Map version : (List>) module.getOrDefault("versions", List.of())) { + versions.putIfAbsent(string(version.get("version")), version); + } + } + } else { + for (Map version : (List>) body.getOrDefault("versions", List.of())) { + versions.putIfAbsent(string(version.get("version")), version); + } + } + } catch (RuntimeException ignored) { + // Continue with healthy members. + } + } + List> sorted = TerraformVersions.descending(versions.keySet()).stream() + .map(versions::get).toList(); + return path.kind() == TerraformPath.Kind.MODULE_VERSIONS + ? json(Map.of("modules", List.of(Map.of( + "source", path.namespace() + "/" + path.name() + "/" + path.system(), "versions", sorted))), headOnly) + : json(Map.of("versions", sorted), headOnly); + } + + @SuppressWarnings("unchecked") + private MavenResponse bindGroupResponse( + RepositoryRuntime group, RepositoryRuntime member, TerraformPath request, MavenResponse response) { + Set pathsToBind = new LinkedHashSet<>(); + if (request.kind() == TerraformPath.Kind.MODULE_DOWNLOAD) { + String value = response.headers().get("X-Terraform-Get"); + if (value != null) pathsToBind.add(canonicalRepositoryPath(value, group.name())); + } else if (request.kind() == TerraformPath.Kind.PROVIDER_DOWNLOAD && response.hasBody()) { + byte[] bytes = responseBytes(response); + Map json = readJson(bytes); + for (String field : List.of("download_url", "shasums_url", "shasums_signature_url")) { + String value = string(json.get(field)); + if (value != null) pathsToBind.add(canonicalRepositoryPath(value, group.name())); + } + // Replace the consumed response body so the client receives the same metadata. + response = bytes(bytes, JSON, false); + } + Instant now = Instant.now(); + for (String bound : pathsToBind) { + if (bound == null || bound.isBlank()) continue; + registry.upsertSourceBinding(new TerraformRegistryDao.SourceBinding( + group.id(), "asset:" + bound, member.id(), memberRevision(member, request), + now.plus(24, ChronoUnit.HOURS), now)); + } + return response; + } + + private long memberRevision(RepositoryRuntime member, TerraformPath path) { + if (path.version() == null) return 0; + return registry.findProviderState(member.id(), path.namespace(), path.name(), path.version()) + .map(TerraformRegistryDao.ProviderState::revision).orElse(0L); + } + + private String remoteUrl(RepositoryRuntime runtime, TerraformPath path) { + String key = path.module() ? "modules.v1" : "providers.v1"; + String service = discovery(runtime, key); + String prefix = path.module() ? "v1/modules/" : "v1/providers/"; + String suffix = path.rawPath().substring(prefix.length()); + return ensureSlash(service) + suffix; + } + + private String discovery(RepositoryRuntime runtime, String key) { + String root = ensureSlash(runtime.proxyRemoteUrl()); + String remote = root + ".well-known/terraform.json"; + String local = ".terraform/upstream/discovery.json"; + Map document = readJson(responseBytes(proxy.getAssetFromUrl(runtime, local, remote, false))); + String value = string(document.get(key)); + if (value == null || value.isBlank()) { + throw new MavenExceptions.BadUpstreamException("Terraform discovery omitted " + key); + } + return URI.create(root).resolve(value).toString(); + } + + private void storeRoute(RepositoryRuntime runtime, String localPath, String remoteUrl, String expectedSha256) { + Map route = new LinkedHashMap<>(); + route.put("remoteUrl", remoteUrl); + if (expectedSha256 != null && !expectedSha256.isBlank()) route.put("sha256", expectedSha256); + assets.storeBytes(runtime, routePath(localPath), jsonBytes(route), JSON, + Map.of("terraformKind", "proxy-route", "targetPath", localPath)); + } + + private static String routePath(String localPath) { + return ".terraform/routes/" + sha256(localPath) + ".json"; + } + + private void enforceWrite(RepositoryRuntime runtime, String path) { + String policy = runtime.writePolicy() == null ? "ALLOW_ONCE" : runtime.writePolicy().toUpperCase(Locale.ROOT); + if ("DENY".equals(policy)) throw new MavenExceptions.WritePolicyDenied("Repository write policy is DENY"); + if ("ALLOW_ONCE".equals(policy) && assets.find(runtime, path).isPresent()) { + throw new MavenExceptions.WritePolicyDenied("Terraform coordinate already exists"); + } + } + + private String filename(String contentDisposition) { + if (contentDisposition == null) { + throw new MavenExceptions.BadRequestException("Content-Disposition filename is required"); + } + Matcher matcher = CONTENT_DISPOSITION_FILENAME.matcher(contentDisposition); + if (!matcher.find()) throw new MavenExceptions.BadRequestException("Content-Disposition filename is required"); + String value = matcher.group(1) == null ? matcher.group(2).trim() : matcher.group(1); + if (matcher.find()) throw new MavenExceptions.BadRequestException("Content-Disposition has duplicate filename values"); + TerraformPathParser.requireFilename(value); + return value; + } + + private MavenResponse json(Map value, boolean headOnly) { + return bytes(jsonBytes(value), JSON, headOnly); + } + + private static MavenResponse bytes(byte[] body, String contentType, boolean headOnly) { + return headOnly + ? MavenResponse.noBody(200, body.length, contentType, null, null) + : MavenResponse.ok(new ByteArrayInputStream(body), body.length, contentType, null, null); + } + + private byte[] jsonBytes(Object value) { + try { return mapper.writeValueAsBytes(value); } + catch (IOException e) { throw new IllegalStateException("Failed rendering Terraform metadata", e); } + } + + private Map readJson(byte[] bytes) { + try { return mapper.readValue(bytes, new TypeReference<>() {}); } + catch (IOException e) { throw new MavenExceptions.BadUpstreamException("Invalid Terraform upstream JSON", e); } + } + + private static byte[] responseBytes(MavenResponse response) { + InputStream body = response.body(); + if (body == null) return new byte[0]; + try (body) { return body.readAllBytes(); } + catch (IOException e) { throw new MavenExceptions.BadUpstreamException("Failed reading Terraform response", e); } + } + + private static List protocolList(String value) { + if (value == null || value.isBlank()) return List.of(PROTOCOLS); + return java.util.Arrays.stream(value.split(",")).map(String::trim).filter(v -> !v.isBlank()).toList(); + } + + private static String safeRemoteFilename(String url, String fallback) { + try { + String path = URI.create(url).getPath(); + String leaf = path == null ? "" : path.substring(path.lastIndexOf('/') + 1); + if (!leaf.isBlank()) { + TerraformPathParser.requireFilename(leaf); + return leaf; + } + } catch (RuntimeException ignored) {} + TerraformPathParser.requireFilename(fallback); + return fallback; + } + + private static String absolute(String base, String value) { + if (value == null || value.isBlank()) throw new MavenExceptions.BadUpstreamException("Terraform upstream URL is missing"); + return URI.create(base).resolve(value).toString(); + } + + private static String repositoryPath(String url, String repository) { + if (url == null) return null; + String marker = "/repository/" + repository + "/"; + int index = url.indexOf(marker); + return index < 0 ? null : url.substring(index + marker.length()); + } + + private String canonicalRepositoryPath(String url, String repository) { + String path = repositoryPath(url, repository); + return path == null ? null : paths.parseRequestPath(path).canonicalPath(); + } + + private static String publicUrl(RequestUrls urls, String assetPath) { + return publicUrl(urls.repositoryBaseUrl(), urls.urlTokenSegment(), assetPath); + } + + static String publicUrl(String repositoryBaseUrl, String urlTokenSegment, String assetPath) { + if (urlTokenSegment == null || urlTokenSegment.isBlank()) { + return repositoryBaseUrl + "/" + assetPath; + } + String prefix; + if (assetPath.startsWith("v1/modules/")) { + prefix = "v1/modules/"; + } else if (assetPath.startsWith("v1/providers/")) { + prefix = "v1/providers/"; + } else { + return repositoryBaseUrl + "/" + assetPath; + } + return repositoryBaseUrl + "/" + prefix + urlTokenSegment + "/" + + assetPath.substring(prefix.length()); + } + + private static String ensureSlash(String value) { + if (value == null || value.isBlank()) throw new MavenExceptions.BadUpstreamException("Terraform remote URL is missing"); + return value.endsWith("/") ? value : value + "/"; + } + + private static String string(Object value) { + return value == null ? null : value.toString(); + } + + private static String sha256(String value) { + try { + return java.util.HexFormat.of().formatHex( + MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8))); + } catch (java.security.NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } + + private static MavenExceptions.MavenNotFoundException notFound(String path) { + return new MavenExceptions.MavenNotFoundException(path); + } + + private record RequestUrls(String repositoryBaseUrl, String urlTokenSegment) {} +} diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformSignatureVerifier.java b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformSignatureVerifier.java new file mode 100644 index 00000000..a8c4f1d8 --- /dev/null +++ b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformSignatureVerifier.java @@ -0,0 +1,68 @@ +package com.github.klboke.kkrepo.server.terraform; + +import com.github.klboke.kkrepo.server.maven.MavenExceptions; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.security.Security; +import java.util.Iterator; +import java.util.List; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.openpgp.PGPCompressedData; +import org.bouncycastle.openpgp.PGPObjectFactory; +import org.bouncycastle.openpgp.PGPPublicKey; +import org.bouncycastle.openpgp.PGPPublicKeyRingCollection; +import org.bouncycastle.openpgp.PGPSignature; +import org.bouncycastle.openpgp.PGPSignatureList; +import org.bouncycastle.openpgp.PGPUtil; +import org.bouncycastle.openpgp.operator.jcajce.JcaKeyFingerprintCalculator; +import org.bouncycastle.openpgp.operator.jcajce.JcaPGPContentVerifierBuilderProvider; +import org.springframework.stereotype.Component; + +/** Verifies upstream detached SHA256SUMS signatures before publishing proxy metadata. */ +@Component +final class TerraformSignatureVerifier { + static { + if (Security.getProvider(BouncyCastleProvider.PROVIDER_NAME) == null) { + Security.addProvider(new BouncyCastleProvider()); + } + } + + void verify(byte[] shasums, byte[] signatureBytes, List publicKeys) { + try { + PGPSignature signature = signature(signatureBytes); + for (String publicKey : publicKeys) { + PGPPublicKeyRingCollection rings = new PGPPublicKeyRingCollection( + PGPUtil.getDecoderStream(new ByteArrayInputStream(publicKey.getBytes(StandardCharsets.UTF_8))), + new JcaKeyFingerprintCalculator()); + PGPPublicKey key = rings.getPublicKey(signature.getKeyID()); + if (key == null) continue; + signature.init(new JcaPGPContentVerifierBuilderProvider() + .setProvider(BouncyCastleProvider.PROVIDER_NAME), key); + signature.update(shasums); + if (signature.verify()) return; + } + throw invalid(); + } catch (MavenExceptions.BadUpstreamException e) { + throw e; + } catch (Exception e) { + throw new MavenExceptions.BadUpstreamException( + "Terraform upstream checksum signature is invalid", e); + } + } + + private static PGPSignature signature(byte[] bytes) throws Exception { + PGPObjectFactory objects = new PGPObjectFactory( + PGPUtil.getDecoderStream(new ByteArrayInputStream(bytes)), new JcaKeyFingerprintCalculator()); + Object value = objects.nextObject(); + if (value instanceof PGPCompressedData compressed) { + objects = new PGPObjectFactory(compressed.getDataStream(), new JcaKeyFingerprintCalculator()); + value = objects.nextObject(); + } + if (value instanceof PGPSignatureList signatures && !signatures.isEmpty()) return signatures.get(0); + throw invalid(); + } + + private static MavenExceptions.BadUpstreamException invalid() { + return new MavenExceptions.BadUpstreamException("Terraform upstream checksum signature is invalid"); + } +} diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformSigningService.java b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformSigningService.java new file mode 100644 index 00000000..330ec8fe --- /dev/null +++ b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformSigningService.java @@ -0,0 +1,164 @@ +package com.github.klboke.kkrepo.server.terraform; + +import com.github.klboke.kkrepo.core.security.EncryptionSecrets; +import com.github.klboke.kkrepo.core.security.SecretCipher; +import com.github.klboke.kkrepo.core.security.TerraformSigningKeyMaterial; +import com.github.klboke.kkrepo.persistence.jdbc.api.TerraformRegistryDao; +import com.github.klboke.kkrepo.server.maven.RepositoryRuntime; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.security.KeyPairGenerator; +import java.security.SecureRandom; +import java.security.Security; +import java.time.Instant; +import java.util.Date; +import java.util.Iterator; +import org.bouncycastle.bcpg.ArmoredOutputStream; +import org.bouncycastle.bcpg.HashAlgorithmTags; +import org.bouncycastle.bcpg.SymmetricKeyAlgorithmTags; +import org.bouncycastle.bcpg.sig.KeyFlags; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.openpgp.PGPKeyPair; +import org.bouncycastle.openpgp.PGPKeyRingGenerator; +import org.bouncycastle.openpgp.PGPPrivateKey; +import org.bouncycastle.openpgp.PGPPublicKey; +import org.bouncycastle.openpgp.PGPSecretKey; +import org.bouncycastle.openpgp.PGPSecretKeyRingCollection; +import org.bouncycastle.openpgp.PGPSignature; +import org.bouncycastle.openpgp.PGPSignatureGenerator; +import org.bouncycastle.openpgp.PGPSignatureSubpacketGenerator; +import org.bouncycastle.openpgp.PGPUtil; +import org.bouncycastle.openpgp.operator.PGPDigestCalculator; +import org.bouncycastle.openpgp.operator.jcajce.JcaKeyFingerprintCalculator; +import org.bouncycastle.openpgp.operator.jcajce.JcaPGPContentSignerBuilder; +import org.bouncycastle.openpgp.operator.jcajce.JcaPGPDigestCalculatorProviderBuilder; +import org.bouncycastle.openpgp.operator.jcajce.JcaPGPKeyPair; +import org.bouncycastle.openpgp.operator.jcajce.JcePBESecretKeyDecryptorBuilder; +import org.bouncycastle.openpgp.operator.jcajce.JcePBESecretKeyEncryptorBuilder; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** Generates repository-scoped OpenPGP keys and creates detached SHA256SUMS signatures. */ +@Service +final class TerraformSigningService { + static { + if (Security.getProvider(BouncyCastleProvider.PROVIDER_NAME) == null) { + Security.addProvider(new BouncyCastleProvider()); + } + } + + private final TerraformRegistryDao registry; + private final TerraformPublishLeaseManager leases; + + TerraformSigningService(TerraformRegistryDao registry) { + this(registry, new TerraformPublishLeaseManager(registry)); + } + + @Autowired + TerraformSigningService(TerraformRegistryDao registry, TerraformPublishLeaseManager leases) { + this.registry = registry; + this.leases = leases; + } + + SigningMaterial active(RepositoryRuntime runtime) { + TerraformRegistryDao.SigningKey row = registry.findActiveSigningKey(runtime.id()).orElse(null); + if (row == null) { + String leaseKey = "signing-key:" + runtime.id(); + try (TerraformPublishLeaseManager.Lease ignored = leases.acquire( + leaseKey, java.time.Duration.ofMinutes(2), java.time.Duration.ofSeconds(30))) { + row = registry.findActiveSigningKey(runtime.id()).orElseGet(() -> create(runtime)); + } + } + String decrypted = new SecretCipher(EncryptionSecrets.credentialSecret()) + .decrypt(row.encryptedPrivateKey()); + TerraformSigningKeyMaterial.Material privateKey = TerraformSigningKeyMaterial.decode(decrypted); + return new SigningMaterial( + row.revision(), row.keyId(), row.publicKey(), privateKey.privateKeyArmor(), privateKey.passphrase()); + } + + TerraformRegistryDao.SigningKey create(RepositoryRuntime runtime) { + TerraformRegistryDao.SigningKey current = registry.findActiveSigningKey(runtime.id()).orElse(null); + if (current != null) return current; + Generated generated = generate(runtime.name()); + String encrypted = new SecretCipher(EncryptionSecrets.credentialSecret()).encrypt( + TerraformSigningKeyMaterial.encode(generated.privateArmor(), "")); + TerraformRegistryDao.SigningKey row = new TerraformRegistryDao.SigningKey( + runtime.id(), 1, generated.keyId(), encrypted, generated.publicArmor(), Instant.now()); + try { + registry.insertSigningKey(row); + return row; + } catch (DuplicateKeyException race) { + return registry.findActiveSigningKey(runtime.id()).orElseThrow(() -> race); + } + } + + byte[] sign(byte[] content, SigningMaterial material) { + try { + PGPSecretKeyRingCollection rings = new PGPSecretKeyRingCollection( + PGPUtil.getDecoderStream(new ByteArrayInputStream(material.privateArmor().getBytes(java.nio.charset.StandardCharsets.UTF_8))), + new JcaKeyFingerprintCalculator()); + PGPSecretKey signing = null; + Iterator ringIterator = rings.getKeyRings(); + while (ringIterator.hasNext() && signing == null) { + Iterator keys = ringIterator.next().getSecretKeys(); + while (keys.hasNext()) { + PGPSecretKey candidate = keys.next(); + if (candidate.isSigningKey()) { signing = candidate; break; } + } + } + if (signing == null) throw new IllegalStateException("Terraform signing key cannot sign"); + PGPPrivateKey privateKey = signing.extractPrivateKey( + new JcePBESecretKeyDecryptorBuilder().setProvider(BouncyCastleProvider.PROVIDER_NAME) + .build(material.passphrase().toCharArray())); + PGPSignatureGenerator generator = new PGPSignatureGenerator( + new JcaPGPContentSignerBuilder(signing.getPublicKey().getAlgorithm(), HashAlgorithmTags.SHA256) + .setProvider(BouncyCastleProvider.PROVIDER_NAME)); + generator.init(PGPSignature.BINARY_DOCUMENT, privateKey); + generator.update(content); + return generator.generate().getEncoded(); + } catch (Exception e) { + throw new IllegalStateException("Failed to sign Terraform provider checksums", e); + } + } + + private static Generated generate(String repositoryName) { + try { + KeyPairGenerator generator = KeyPairGenerator.getInstance("RSA", BouncyCastleProvider.PROVIDER_NAME); + generator.initialize(3072, new SecureRandom()); + PGPKeyPair pair = new JcaPGPKeyPair(PGPPublicKey.RSA_SIGN, generator.generateKeyPair(), new Date()); + PGPDigestCalculator sha1 = new JcaPGPDigestCalculatorProviderBuilder() + .setProvider(BouncyCastleProvider.PROVIDER_NAME).build().get(HashAlgorithmTags.SHA1); + PGPSignatureSubpacketGenerator certification = new PGPSignatureSubpacketGenerator(); + certification.setKeyFlags(false, KeyFlags.CERTIFY_OTHER | KeyFlags.SIGN_DATA); + PGPKeyRingGenerator rings = new PGPKeyRingGenerator( + PGPSignature.POSITIVE_CERTIFICATION, + pair, + "kkrepo Terraform ", + sha1, + certification.generate(), + null, + new JcaPGPContentSignerBuilder(pair.getPublicKey().getAlgorithm(), HashAlgorithmTags.SHA256) + .setProvider(BouncyCastleProvider.PROVIDER_NAME), + new JcePBESecretKeyEncryptorBuilder(SymmetricKeyAlgorithmTags.AES_256, sha1) + .setProvider(BouncyCastleProvider.PROVIDER_NAME).build(new char[0])); + ByteArrayOutputStream publicBytes = new ByteArrayOutputStream(); + try (ArmoredOutputStream armor = new ArmoredOutputStream(publicBytes)) { + rings.generatePublicKeyRing().encode(armor); + } + ByteArrayOutputStream privateBytes = new ByteArrayOutputStream(); + try (ArmoredOutputStream armor = new ArmoredOutputStream(privateBytes)) { + rings.generateSecretKeyRing().encode(armor); + } + String keyId = String.format("%016X", pair.getKeyID()); + return new Generated(keyId, publicBytes.toString(java.nio.charset.StandardCharsets.UTF_8), + privateBytes.toString(java.nio.charset.StandardCharsets.UTF_8)); + } catch (Exception e) { + throw new IllegalStateException("Failed to generate Terraform repository signing key", e); + } + } + + record SigningMaterial( + int revision, String keyId, String publicArmor, String privateArmor, String passphrase) {} + private record Generated(String keyId, String publicArmor, String privateArmor) {} +} diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/upload/ComponentUploadService.java b/server/src/main/java/com/github/klboke/kkrepo/server/upload/ComponentUploadService.java index 5dc6acbc..6fcf1df8 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/upload/ComponentUploadService.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/upload/ComponentUploadService.java @@ -16,6 +16,8 @@ import com.github.klboke.kkrepo.server.pypi.PypiHostedService; import com.github.klboke.kkrepo.server.pub.PubHostedService; import com.github.klboke.kkrepo.server.raw.RawHostedService; +import com.github.klboke.kkrepo.server.terraform.TerraformService; +import com.github.klboke.kkrepo.protocol.terraform.TerraformPathParser; import com.github.klboke.kkrepo.server.yum.YumService; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -65,6 +67,18 @@ public class ComponentUploadService { field("version", "STRING", "Package version override", true, "Component coordinates")), List.of(field("asset", "FILE", "Composer archive", false, null))), + new UploadDefinition( + "terraform", + false, + List.of( + field("kind", "STRING", "module or provider", false, "Component coordinates"), + field("namespace", "STRING", "Namespace", false, "Component coordinates"), + field("name", "STRING", "Module name or provider type", false, "Component coordinates"), + field("version", "STRING", "Semantic version", false, "Component coordinates"), + field("system", "STRING", "Module target system", true, "Component coordinates"), + field("os", "STRING", "Provider operating system", true, "Component coordinates"), + field("arch", "STRING", "Provider architecture", true, "Component coordinates")), + List.of(field("asset", "FILE", "Terraform module/provider archive", false, null))), rawLikeUpload("nuget"), rawLikeUpload("rubygems"), rawLikeUpload("yum"), @@ -81,7 +95,9 @@ public class ComponentUploadService { private final ComposerHostedService composerHosted; private final RawHostedService rawHosted; private final YumService yumService; + private final TerraformService terraformService; private final MavenPathParser mavenPathParser = new MavenPathParser(); + private final TerraformPathParser terraformPathParser = new TerraformPathParser(); @Autowired public ComponentUploadService( @@ -95,7 +111,8 @@ public ComponentUploadService( PubHostedService pubHosted, ComposerHostedService composerHosted, RawHostedService rawHosted, - YumService yumService) { + YumService yumService, + TerraformService terraformService) { this.registry = registry; this.assetDao = assetDao; this.mavenHosted = mavenHosted; @@ -107,6 +124,16 @@ public ComponentUploadService( this.composerHosted = composerHosted; this.rawHosted = rawHosted; this.yumService = yumService; + this.terraformService = terraformService; + } + + public ComponentUploadService( + RepositoryRuntimeRegistry registry, AssetDao assetDao, MavenHostedService mavenHosted, + NpmHostedService npmHosted, PypiHostedService pypiHosted, HelmHostedService helmHosted, + CargoHostedService cargoHosted, PubHostedService pubHosted, ComposerHostedService composerHosted, + RawHostedService rawHosted, YumService yumService) { + this(registry, assetDao, mavenHosted, npmHosted, pypiHosted, helmHosted, cargoHosted, + pubHosted, composerHosted, rawHosted, yumService, null); } /** Backward-compatible constructor used by existing focused upload tests. */ @@ -122,7 +149,7 @@ public ComponentUploadService( RawHostedService rawHosted, YumService yumService) { this(registry, assetDao, mavenHosted, npmHosted, pypiHosted, helmHosted, cargoHosted, - pubHosted, null, rawHosted, yumService); + pubHosted, null, rawHosted, yumService, null); } public List definitions() { @@ -162,6 +189,7 @@ public UploadResult upload( case CARGO -> uploadCargo(runtime, upload, createdBy, createdByIp); case PUB -> uploadPub(runtime, upload, createdBy, createdByIp); case COMPOSER -> uploadComposer(runtime, upload, createdBy, createdByIp); + case TERRAFORM -> uploadTerraform(runtime, upload, createdBy, createdByIp); case DOCKER -> throw new UploadValidationException("Docker hosted upload must use the Docker Registry V2 API"); case NUGET -> uploadRaw(runtime, upload, createdBy, createdByIp); case RUBYGEMS -> uploadRaw(runtime, upload, createdBy, createdByIp); @@ -171,6 +199,37 @@ public UploadResult upload( return new UploadResult(paths); } + private List uploadTerraform( + RepositoryRuntime runtime, NormalizedUpload upload, String createdBy, String createdByIp) throws IOException { + if (terraformService == null) throw new UploadValidationException("Terraform upload service is unavailable"); + String kind = requireField(upload.fields(), "kind").trim().toLowerCase(Locale.ROOT); + String namespace = requireField(upload.fields(), "namespace"); + String name = requireField(upload.fields(), "name"); + String version = requireField(upload.fields(), "version"); + AssetUpload asset = upload.assets().stream().findFirst() + .orElseThrow(() -> new UploadValidationException("Terraform upload requires one archive")); + String filename = asset.file().getOriginalFilename(); + if (filename == null || filename.isBlank()) filename = "terraform.zip"; + String path; + String disposition = null; + if ("module".equals(kind)) { + String system = requireField(upload.fields(), "system"); + path = "v1/modules/" + namespace + "/" + name + "/" + system + "/" + version + "/" + filename; + } else if ("provider".equals(kind)) { + String os = requireField(upload.fields(), "os"); + String arch = requireField(upload.fields(), "arch"); + path = "v1/providers/" + namespace + "/" + name + "/" + version + "/download/" + os + "/" + arch; + disposition = "attachment; filename=\"" + filename.replace("\"", "") + "\""; + } else { + throw new UploadValidationException("terraform.kind must be module or provider"); + } + try (var body = asset.file().getInputStream()) { + terraformService.put(runtime, terraformPathParser.parse(path), body, asset.file().getContentType(), + disposition, createdBy, createdByIp); + } + return List.of(path); + } + private List uploadMaven( RepositoryRuntime runtime, NormalizedUpload upload, @@ -441,6 +500,7 @@ private static String formatLabel(RepositoryFormat format) { case NUGET -> "nuget"; case RUBYGEMS -> "rubygems"; case YUM -> "yum"; + case TERRAFORM -> "terraform"; case RAW -> "raw"; }; } diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/metrics/RepositoryRequestMetricsFilterTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/metrics/RepositoryRequestMetricsFilterTest.java index eced1d7b..a779a13d 100644 --- a/server/src/test/java/com/github/klboke/kkrepo/server/metrics/RepositoryRequestMetricsFilterTest.java +++ b/server/src/test/java/com/github/klboke/kkrepo/server/metrics/RepositoryRequestMetricsFilterTest.java @@ -180,6 +180,40 @@ void logsNonSuccessRepositoryRequestDetailsWhenEnabled() throws Exception { assertFalse(message.contains("secret-token")); } + @Test + void stripsTerraformUrlTokenBeforeOperationMetricsAndFailureLogs() throws Exception { + SimpleMeterRegistry registry = new SimpleMeterRegistry(); + RepositoryRequestMetricsFilter filter = new RepositoryRequestMetricsFilter(new KkRepoMetrics(registry), true, ""); + String token = "GenericToken.super-secret-value"; + String canonical = "v1/providers/kkrepo/fixture/versions"; + MockHttpServletRequest request = new MockHttpServletRequest( + "GET", "/repository/terraform-group/v1/providers/" + token + "/kkrepo/fixture/versions"); + MockHttpServletResponse response = new MockHttpServletResponse(); + FilterChain chain = (req, resp) -> { + req.setAttribute( + RepositorySecurityFilter.REPOSITORY_RECORD_ATTRIBUTE, + repository("terraform-group", RepositoryFormat.TERRAFORM, RepositoryType.GROUP)); + req.setAttribute(RepositorySecurityFilter.NORMALIZED_REPOSITORY_PATH_ATTRIBUTE, canonical); + ((MockHttpServletResponse) resp).sendError(404); + }; + + ListAppender appender = attachAppender(); + try { + filter.doFilter(request, response, chain); + } finally { + detachAppender(appender); + } + + var counter = registry.find("kkrepo_repository_requests_total") + .tags("operation", "terraform_provider_versions", "status", "404") + .counter(); + assertNotNull(counter); + String message = appender.list.get(0).getFormattedMessage(); + assertTrue(message.contains("uri=/repository/terraform-group/" + canonical)); + assertTrue(message.contains("path=" + canonical)); + assertFalse(message.contains(token)); + } + @Test void doesNotLogNonSuccessRepositoryRequestDetailsWhenDisabled() throws Exception { SimpleMeterRegistry registry = new SimpleMeterRegistry(); diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/migration/NexusMigrationControllerTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/migration/NexusMigrationControllerTest.java index 432e93fe..a2de081e 100644 --- a/server/src/test/java/com/github/klboke/kkrepo/server/migration/NexusMigrationControllerTest.java +++ b/server/src/test/java/com/github/klboke/kkrepo/server/migration/NexusMigrationControllerTest.java @@ -188,6 +188,7 @@ void repositoryDataRequestRequiresSourcePassword() throws Exception { null, null, null, + null, null); MockHttpServletRequest request = new MockHttpServletRequest( "POST", @@ -318,6 +319,7 @@ private static NexusMigrationController controllerWith(BlobStoreRecord defaultSt null, null, null, + null, null); } @@ -432,7 +434,8 @@ private TestableNexusMigrationController( securityAuthorizationCache, apiKeyAuthCache, basicAuthCache, - dockerConnectorRuntime); + dockerConnectorRuntime, + null); this.migrationService = migrationService; } diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/security/RepositorySecurityFilterTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/security/RepositorySecurityFilterTest.java index e7ade3cb..a2b0fd14 100644 --- a/server/src/test/java/com/github/klboke/kkrepo/server/security/RepositorySecurityFilterTest.java +++ b/server/src/test/java/com/github/klboke/kkrepo/server/security/RepositorySecurityFilterTest.java @@ -180,6 +180,36 @@ void pubChallengeUsesBearerRealm() throws Exception { response.headers.get("WWW-Authenticate")); } + @Test + void terraformUrlTokenIsAuthenticatedButCanonicalPathIsUsedForAuthorization() throws Exception { + StubAuthenticationService authentication = new StubAuthenticationService(Optional.empty()); + authentication.terraformAuthenticated = Optional.of(subject("alice")); + RecordingDecisionService decisions = new RecordingDecisionService(AccessDecision.allow()); + RepositorySecurityFilter filter = filter( + authentication, + decisions, + new FakeRepositoryDao(repository( + "terraform-private", RepositoryFormat.TERRAFORM, RepositoryType.HOSTED)), + false); + ResponseState response = new ResponseState(); + ChainState chain = new ChainState(); + HttpServletRequest request = request( + "GET", + "/repository/terraform-private/v1/modules/YWRtaW46QWRtaW4xMjM0/kkrepo/fixture/aws/versions"); + + filter.doFilter(request, response.proxy(), chain); + + assertEquals(1, chain.calls); + assertEquals("YWRtaW46QWRtaW4xMjM0", authentication.terraformToken); + assertEquals( + "YWRtaW46QWRtaW4xMjM0", + request.getAttribute(RepositorySecurityFilter.TERRAFORM_URL_TOKEN_SEGMENT_ATTRIBUTE)); + assertEquals( + "v1/modules/kkrepo/fixture/aws/versions", + request.getAttribute(RepositorySecurityFilter.NORMALIZED_REPOSITORY_PATH_ATTRIBUTE)); + assertEquals("v1/modules/kkrepo/fixture/aws/versions", decisions.permission.pathPattern()); + } + @Test void cargoSparseIndexRequiresAuthenticationWhenAnonymousReadIsDisabled() throws Exception { assertCargoReadRequiresAuthenticationWhenAnonymousAccessDisabled("/repository/cargo-hosted/kk/re/kkrepo_e2e"); @@ -882,6 +912,8 @@ private static class StubAuthenticationService extends SecurityAuthenticationSer private int rubygemsCalls; private Optional pubAuthenticated = Optional.empty(); private int pubCalls; + private Optional terraformAuthenticated = Optional.empty(); + private String terraformToken; private boolean anonymousEnabled; private StubAuthenticationService(AuthenticatedSubject anonymous) { @@ -916,6 +948,12 @@ public Optional authenticatePub(HttpServletRequest request return pubAuthenticated; } + @Override + public Optional authenticateTerraformUrlToken(String token) { + terraformToken = token; + return terraformAuthenticated; + } + @Override public Optional authenticateAnonymous() { anonymousCalls++; diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformArchiveInspectorTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformArchiveInspectorTest.java new file mode 100644 index 00000000..c2a2ebc1 --- /dev/null +++ b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformArchiveInspectorTest.java @@ -0,0 +1,75 @@ +package com.github.klboke.kkrepo.server.terraform; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.github.klboke.kkrepo.server.maven.MavenExceptions; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.zip.GZIPOutputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; +import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; +import org.junit.jupiter.api.Test; + +class TerraformArchiveInspectorTest { + private final TerraformArchiveInspector inspector = new TerraformArchiveInspector(); + + @Test + void acceptsEveryNexusDocumentedTarCompressionVariant() throws Exception { + byte[] source = "terraform {}\n".getBytes(java.nio.charset.StandardCharsets.UTF_8); + for (String suffix : List.of(".tar.gz", ".tgz", ".tar.xz", ".txz", ".xz", ".tar.bz2", ".tbz2")) { + Path file = inspector.bufferAndInspect( + new ByteArrayInputStream(tar(source, suffix)), "module" + suffix, true, null); + assertTrue(Files.size(file) > 0, suffix); + Files.delete(file); + } + } + + @Test + void rejectsTraversalAndUnsupportedCompression() throws Exception { + byte[] traversal = tarEntry("../escape.tf", "terraform {}".getBytes(), ".tar.gz"); + assertThrows(MavenExceptions.BadRequestException.class, + () -> inspector.bufferAndInspect(new ByteArrayInputStream(traversal), "module.tar.gz", true, null)); + assertThrows(MavenExceptions.BadRequestException.class, + () -> inspector.bufferAndInspect(new ByteArrayInputStream(new byte[0]), "module.tar", true, null)); + } + + @Test + void rejectsArchiveWithBombLikeExpansionRatio() throws Exception { + byte[] zeros = new byte[2 * 1024 * 1024]; + byte[] archive = tar(zeros, ".tar.gz"); + assertThrows(MavenExceptions.BadRequestException.class, + () -> inspector.bufferAndInspect(new ByteArrayInputStream(archive), "module.tar.gz", true, null)); + } + + private static byte[] tar(byte[] content, String suffix) throws IOException { + return tarEntry("fixture/main.tf", content, suffix); + } + + private static byte[] tarEntry(String name, byte[] content, String suffix) throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (OutputStream compressed = compressor(bytes, suffix); + TarArchiveOutputStream tar = new TarArchiveOutputStream(compressed)) { + TarArchiveEntry entry = new TarArchiveEntry(name); + entry.setSize(content.length); + tar.putArchiveEntry(entry); + tar.write(content); + tar.closeArchiveEntry(); + tar.finish(); + } + return bytes.toByteArray(); + } + + private static OutputStream compressor(OutputStream out, String suffix) throws IOException { + if (suffix.endsWith("gz") || suffix.endsWith("tgz")) return new GZIPOutputStream(out); + if (suffix.endsWith("xz") || suffix.endsWith("txz")) return new XZCompressorOutputStream(out); + return new BZip2CompressorOutputStream(out); + } +} diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformPublishLeaseManagerTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformPublishLeaseManagerTest.java new file mode 100644 index 00000000..a70e95a0 --- /dev/null +++ b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformPublishLeaseManagerTest.java @@ -0,0 +1,36 @@ +package com.github.klboke.kkrepo.server.terraform; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.github.klboke.kkrepo.persistence.jdbc.api.TerraformRegistryDao; +import java.lang.reflect.Proxy; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; + +class TerraformPublishLeaseManagerTest { + @Test + void waitsForAnotherReplicaAndReleasesOnlyOnce() { + AtomicInteger attempts = new AtomicInteger(); + AtomicInteger releases = new AtomicInteger(); + TerraformRegistryDao dao = (TerraformRegistryDao) Proxy.newProxyInstance( + TerraformRegistryDao.class.getClassLoader(), + new Class[] {TerraformRegistryDao.class}, + (proxy, method, args) -> switch (method.getName()) { + case "tryAcquirePublishLease" -> attempts.incrementAndGet() >= 3; + case "releasePublishLease" -> { + releases.incrementAndGet(); + yield null; + } + default -> throw new UnsupportedOperationException(method.getName()); + }); + TerraformPublishLeaseManager manager = new TerraformPublishLeaseManager(dao); + + TerraformPublishLeaseManager.Lease lease = manager.acquire( + "provider:1", java.time.Duration.ofMinutes(1), java.time.Duration.ofSeconds(1)); + lease.close(); + lease.close(); + + assertEquals(3, attempts.get()); + assertEquals(1, releases.get()); + } +} diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformRepositoryDataMigrationWriterTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformRepositoryDataMigrationWriterTest.java new file mode 100644 index 00000000..52735edd --- /dev/null +++ b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformRepositoryDataMigrationWriterTest.java @@ -0,0 +1,23 @@ +package com.github.klboke.kkrepo.server.terraform; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class TerraformRepositoryDataMigrationWriterTest { + @Test + void selectsOnlyNexusModuleAndProviderArchivesForMigration() { + assertTrue(TerraformRepositoryDataMigrationWriter.isMigratableTerraformPath( + "/v1/modules/kkrepo/network/aws/1.2.3/kkrepo-network-aws_1.2.3.zip")); + assertTrue(TerraformRepositoryDataMigrationWriter.isMigratableTerraformPath( + "/v1/providers/kkrepo/fixture/1.2.3/download/linux/amd64/terraform-provider-fixture_1.2.3_linux_amd64.zip")); + + assertFalse(TerraformRepositoryDataMigrationWriter.isMigratableTerraformPath( + "/v1/providers/kkrepo/fixture/1.2.3/download/linux/amd64/SHA256SUMS")); + assertFalse(TerraformRepositoryDataMigrationWriter.isMigratableTerraformPath( + "/v1/providers/kkrepo/fixture/versions.json")); + assertFalse(TerraformRepositoryDataMigrationWriter.isMigratableTerraformPath( + "/v1/providers/kkrepo/fixture/1.2.3/download/linux/amd64/../../escape.zip")); + } +} diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceUrlTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceUrlTest.java new file mode 100644 index 00000000..b92ae725 --- /dev/null +++ b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceUrlTest.java @@ -0,0 +1,31 @@ +package com.github.klboke.kkrepo.server.terraform; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class TerraformServiceUrlTest { + private static final String BASE = "https://repo.example/repository/terraform-private"; + + @Test + void keepsUrlTokenOnModuleAndProviderFollowUpUrls() { + assertEquals( + BASE + "/v1/modules/dXNlcjpwYXNz/acme/network/aws/1.2.3/module.zip", + TerraformService.publicUrl( + BASE, "dXNlcjpwYXNz", "v1/modules/acme/network/aws/1.2.3/module.zip")); + assertEquals( + BASE + "/v1/providers/dXNlcjpwYXNz/acme/cloud/1.2.3/package/linux/provider.zip", + TerraformService.publicUrl( + BASE, "dXNlcjpwYXNz", "v1/providers/acme/cloud/1.2.3/package/linux/provider.zip")); + } + + @Test + void leavesCanonicalUrlsAndInternalPathsUnchanged() { + assertEquals( + BASE + "/v1/modules/acme/network/aws/versions", + TerraformService.publicUrl(BASE, null, "v1/modules/acme/network/aws/versions")); + assertEquals( + BASE + "/.terraform/routes/value.json", + TerraformService.publicUrl(BASE, "dXNlcjpwYXNz", ".terraform/routes/value.json")); + } +} diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformSigningServiceTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformSigningServiceTest.java new file mode 100644 index 00000000..9de20ca1 --- /dev/null +++ b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformSigningServiceTest.java @@ -0,0 +1,66 @@ +package com.github.klboke.kkrepo.server.terraform; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.github.klboke.kkrepo.core.RepositoryFormat; +import com.github.klboke.kkrepo.core.RepositoryType; +import com.github.klboke.kkrepo.persistence.jdbc.api.TerraformRegistryDao; +import com.github.klboke.kkrepo.server.maven.RepositoryRuntime; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; +import org.bouncycastle.openpgp.PGPObjectFactory; +import org.bouncycastle.openpgp.PGPPublicKeyRingCollection; +import org.bouncycastle.openpgp.PGPSignatureList; +import org.bouncycastle.openpgp.PGPUtil; +import org.bouncycastle.openpgp.operator.jcajce.JcaKeyFingerprintCalculator; +import org.bouncycastle.openpgp.operator.jcajce.JcaPGPContentVerifierBuilderProvider; +import org.junit.jupiter.api.Test; + +class TerraformSigningServiceTest { + @Test + void generatedDetachedSignatureVerifiesWithPublishedPublicKey() throws Exception { + TerraformRegistryDao dao = mock(TerraformRegistryDao.class); + AtomicReference stored = new AtomicReference<>(); + when(dao.findActiveSigningKey(7L)).thenAnswer(ignored -> Optional.ofNullable(stored.get())); + when(dao.tryAcquirePublishLease(anyString(), anyString(), any(Instant.class))).thenReturn(true); + doAnswer(invocation -> { stored.set(invocation.getArgument(0)); return null; }) + .when(dao).insertSigningKey(any()); + TerraformSigningService service = new TerraformSigningService(dao); + RepositoryRuntime runtime = new RepositoryRuntime( + 7L, "terraform-hosted", RepositoryFormat.TERRAFORM, RepositoryType.HOSTED, + "terraform-hosted", true, 1L, "ALLOW_ONCE", null, null, true, + null, null, null, List.of()); + byte[] contents = "abcd terraform-provider-demo_1.0.0_linux_amd64.zip\n" + .getBytes(StandardCharsets.UTF_8); + TerraformSigningService.SigningMaterial material = service.active(runtime); + byte[] detached = service.sign(contents, material); + + new TerraformSignatureVerifier().verify(contents, detached, List.of(material.publicArmor())); + + PGPPublicKeyRingCollection publicKeys = new PGPPublicKeyRingCollection( + PGPUtil.getDecoderStream(new ByteArrayInputStream(material.publicArmor().getBytes(StandardCharsets.UTF_8))), + new JcaKeyFingerprintCalculator()); + PGPObjectFactory objects = new PGPObjectFactory( + PGPUtil.getDecoderStream(new ByteArrayInputStream(detached)), new JcaKeyFingerprintCalculator()); + var signature = ((PGPSignatureList) objects.nextObject()).get(0); + assertEquals(material.keyId(), String.format("%016X", signature.getKeyID())); + var primary = publicKeys.getPublicKey(signature.getKeyID()); + var identities = primary.getSignaturesForID(primary.getUserIDs().next()); + var certification = identities.next(); + assertTrue(certification.getHashedSubPackets().getKeyFlags() != 0); + signature.init(new JcaPGPContentVerifierBuilderProvider().setProvider("BC"), + primary); + signature.update(contents); + assertTrue(signature.verify()); + } +} From 84ad2a822ac61f08c4045504e8f79a1ead147203 Mon Sep 17 00:00:00 2001 From: kl Date: Wed, 15 Jul 2026 00:49:17 +0800 Subject: [PATCH 02/53] fix: parse Terraform upload filenames in linear time --- .../server/terraform/TerraformService.java | 73 +++++++++++++++++-- .../terraform/TerraformServiceUrlTest.java | 36 +++++++++ 2 files changed, 101 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java index c018586b..4c47720c 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java @@ -33,8 +33,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.springframework.http.MediaType; import org.springframework.stereotype.Service; @@ -44,8 +42,6 @@ public class TerraformService { private static final String JSON = MediaType.APPLICATION_JSON_VALUE; private static final String OCTET = MediaType.APPLICATION_OCTET_STREAM_VALUE; private static final String PROTOCOLS = "5.0"; - private static final Pattern CONTENT_DISPOSITION_FILENAME = Pattern.compile( - "(?i)(?:^|;)\\s*filename\\s*=\\s*(?:\"([^\"]+)\"|([^;]+))\\s*(?:;|$)"); private final ObjectMapper mapper; private final TerraformAssetSupport assets; @@ -577,17 +573,78 @@ private void enforceWrite(RepositoryRuntime runtime, String path) { } private String filename(String contentDisposition) { + return contentDispositionFilename(contentDisposition); + } + + static String contentDispositionFilename(String contentDisposition) { if (contentDisposition == null) { throw new MavenExceptions.BadRequestException("Content-Disposition filename is required"); } - Matcher matcher = CONTENT_DISPOSITION_FILENAME.matcher(contentDisposition); - if (!matcher.find()) throw new MavenExceptions.BadRequestException("Content-Disposition filename is required"); - String value = matcher.group(1) == null ? matcher.group(2).trim() : matcher.group(1); - if (matcher.find()) throw new MavenExceptions.BadRequestException("Content-Disposition has duplicate filename values"); + String value = null; + boolean filenameSeen = false; + boolean quoted = false; + boolean escaped = false; + int parameterStart = 0; + for (int i = 0; i <= contentDisposition.length(); i++) { + char current = i == contentDisposition.length() ? ';' : contentDisposition.charAt(i); + if (escaped) { + escaped = false; + } else if (quoted && current == '\\') { + escaped = true; + } else if (current == '"') { + quoted = !quoted; + } else if (current == ';' && !quoted) { + String parameter = contentDisposition.substring(parameterStart, i).trim(); + parameterStart = i + 1; + int equals = parameter.indexOf('='); + if (equals <= 0 || !"filename".equalsIgnoreCase(parameter.substring(0, equals).trim())) { + continue; + } + if (filenameSeen) { + throw new MavenExceptions.BadRequestException( + "Content-Disposition has duplicate filename values"); + } + filenameSeen = true; + value = dispositionParameterValue(parameter.substring(equals + 1).trim()); + } + } + if (quoted || escaped) { + throw new MavenExceptions.BadRequestException("Content-Disposition has an invalid quoted value"); + } + if (!filenameSeen) { + throw new MavenExceptions.BadRequestException("Content-Disposition filename is required"); + } TerraformPathParser.requireFilename(value); return value; } + private static String dispositionParameterValue(String raw) { + if (!raw.startsWith("\"")) return raw; + if (raw.length() < 2 || raw.charAt(raw.length() - 1) != '"') { + throw new MavenExceptions.BadRequestException("Content-Disposition has an invalid quoted value"); + } + StringBuilder decoded = new StringBuilder(raw.length() - 2); + boolean escaped = false; + for (int i = 1; i < raw.length() - 1; i++) { + char current = raw.charAt(i); + if (escaped) { + decoded.append(current); + escaped = false; + } else if (current == '\\') { + escaped = true; + } else if (current == '"') { + throw new MavenExceptions.BadRequestException( + "Content-Disposition has an invalid quoted value"); + } else { + decoded.append(current); + } + } + if (escaped) { + throw new MavenExceptions.BadRequestException("Content-Disposition has an invalid quoted value"); + } + return decoded.toString(); + } + private MavenResponse json(Map value, boolean headOnly) { return bytes(jsonBytes(value), JSON, headOnly); } diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceUrlTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceUrlTest.java index b92ae725..6f3546e6 100644 --- a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceUrlTest.java +++ b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceUrlTest.java @@ -1,7 +1,9 @@ package com.github.klboke.kkrepo.server.terraform; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import com.github.klboke.kkrepo.server.maven.MavenExceptions; import org.junit.jupiter.api.Test; class TerraformServiceUrlTest { @@ -28,4 +30,38 @@ void leavesCanonicalUrlsAndInternalPathsUnchanged() { BASE + "/.terraform/routes/value.json", TerraformService.publicUrl(BASE, "dXNlcjpwYXNz", ".terraform/routes/value.json")); } + + @Test + void parsesContentDispositionFilenameWithQuotedSeparatorsAndEscapes() { + assertEquals( + "terraform-provider-cloud_1.2.3_linux_amd64.zip", + TerraformService.contentDispositionFilename( + "attachment; name=archive; filename=terraform-provider-cloud_1.2.3_linux_amd64.zip")); + assertEquals( + "provider;linux.zip", + TerraformService.contentDispositionFilename( + "attachment; filename=\"provider;linux.zip\"; ignored=\"a;b\"")); + assertEquals( + "provider-linux.zip", + TerraformService.contentDispositionFilename( + "attachment; filename=\"provider\\-linux.zip\"")); + } + + @Test + void rejectsMissingDuplicateAndMalformedContentDispositionFilename() { + assertThrows(MavenExceptions.BadRequestException.class, + () -> TerraformService.contentDispositionFilename("attachment; name=archive")); + assertThrows(MavenExceptions.BadRequestException.class, + () -> TerraformService.contentDispositionFilename( + "attachment; filename=one.zip; FILENAME=two.zip")); + assertThrows(MavenExceptions.BadRequestException.class, + () -> TerraformService.contentDispositionFilename( + "attachment; filename=\"unterminated.zip")); + } + + @Test + void parsesLongUntrustedWhitespaceInLinearTime() { + String header = "attachment; filename=" + " ".repeat(100_000) + "provider.zip"; + assertEquals("provider.zip", TerraformService.contentDispositionFilename(header)); + } } From b956f150f2dd84446647eeffe98ca4d67677a523 Mon Sep 17 00:00:00 2001 From: kl Date: Wed, 15 Jul 2026 01:05:46 +0800 Subject: [PATCH 03/53] test: cover Terraform repository service workflows --- .../terraform/TerraformAssetSupportTest.java | 125 ++++++ .../terraform/TerraformServiceTest.java | 385 ++++++++++++++++++ 2 files changed, 510 insertions(+) create mode 100644 server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformAssetSupportTest.java create mode 100644 server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformAssetSupportTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformAssetSupportTest.java new file mode 100644 index 00000000..5074f220 --- /dev/null +++ b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformAssetSupportTest.java @@ -0,0 +1,125 @@ +package com.github.klboke.kkrepo.server.terraform; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.github.klboke.kkrepo.core.BlobObjectMetadata; +import com.github.klboke.kkrepo.core.BlobReference; +import com.github.klboke.kkrepo.core.BlobStorage; +import com.github.klboke.kkrepo.core.RepositoryFormat; +import com.github.klboke.kkrepo.core.RepositoryType; +import com.github.klboke.kkrepo.persistence.jdbc.api.AssetDao; +import com.github.klboke.kkrepo.persistence.jdbc.api.model.AssetBlobRecord; +import com.github.klboke.kkrepo.persistence.jdbc.api.model.AssetRecord; +import com.github.klboke.kkrepo.server.maven.BlobStorageRegistry; +import com.github.klboke.kkrepo.server.maven.MavenExceptions; +import com.github.klboke.kkrepo.server.maven.MavenResponse; +import com.github.klboke.kkrepo.server.maven.RepositoryRuntime; +import com.github.klboke.kkrepo.server.raw.RawHostedService; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class TerraformAssetSupportTest { + private AssetDao assetDao; + private BlobStorageRegistry storages; + private RawHostedService hosted; + private BlobStorage storage; + private TerraformAssetSupport support; + private RepositoryRuntime runtime; + private AssetRecord asset; + private AssetBlobRecord blob; + + @BeforeEach + void setUp() { + assetDao = mock(AssetDao.class); + storages = mock(BlobStorageRegistry.class); + hosted = mock(RawHostedService.class); + storage = mock(BlobStorage.class); + support = new TerraformAssetSupport(assetDao, storages, hosted); + runtime = new RepositoryRuntime( + 1, "terraform", RepositoryFormat.TERRAFORM, RepositoryType.HOSTED, + "terraform-hosted", true, 1L, "ALLOW_ONCE", null, null, true, + null, null, null, List.of()); + asset = new AssetRecord( + 2L, runtime.id(), null, 3L, RepositoryFormat.TERRAFORM, "v1/file.zip", + new byte[32], "file.zip", "terraform", "application/zip", 7L, + null, Instant.parse("2026-07-14T00:00:00Z"), Map.of()); + blob = new AssetBlobRecord( + 3L, 4L, "bucket/object", new byte[32], "object", new byte[32], + "sha1", "sha256", "md5", 7, "application/zip", "alice", null, + Instant.now(), Instant.now(), Map.of()); + } + + @Test + void delegatesStoresAndDatabaseQueries() { + support.store(runtime, "v1/file.zip", new ByteArrayInputStream(new byte[0]), + "application/zip", Map.of("kind", "archive"), "alice", "127.0.0.1"); + support.storeBytes(runtime, "v1/metadata.json", "{}".getBytes(StandardCharsets.UTF_8), + "application/json", Map.of("kind", "metadata")); + verify(hosted).putInternal(eq(runtime), eq("v1/file.zip"), any(), + eq("application/zip"), any(), eq("alice"), eq("127.0.0.1")); + verify(hosted).putInternal(eq(runtime), eq("v1/metadata.json"), any(), + eq("application/json"), any(), eq("terraform"), eq(null)); + + when(assetDao.findAssetByPath(runtime.id(), asset.path())).thenReturn(Optional.of(asset)); + when(assetDao.listAssetsByPrefix(runtime.id(), "v1/")).thenReturn(List.of(asset)); + when(assetDao.findBlobById(3L)).thenReturn(Optional.of(blob)); + assertEquals(Optional.of(asset), support.find(runtime, asset.path())); + assertEquals(List.of(asset), support.list(runtime, "v1/")); + assertEquals(blob, support.blob(asset)); + assertEquals(null, support.blob(new AssetRecord( + 5L, runtime.id(), null, null, RepositoryFormat.TERRAFORM, "missing", new byte[0], + "missing", "terraform", null, null, null, null, Map.of()))); + } + + @Test + void readsAndServesAssetsThroughConfiguredBlobStorage() throws Exception { + byte[] content = "archive".getBytes(StandardCharsets.UTF_8); + when(assetDao.findAssetByPath(runtime.id(), asset.path())).thenReturn(Optional.of(asset)); + when(assetDao.findBlobById(3L)).thenReturn(Optional.of(blob)); + when(storages.forBlobStoreId(blob.blobStoreId())).thenReturn(storage); + when(storage.get(any())).thenAnswer(invocation -> + Optional.of(new ByteArrayInputStream(content))); + when(storage.stat(any())).thenAnswer(invocation -> { + BlobReference reference = invocation.getArgument(0); + return Optional.of(new BlobObjectMetadata(reference, "etag", "application/zip", Instant.now())); + }); + + assertArrayEquals(content, support.bytes(runtime, asset.path())); + MavenResponse head = support.serve(runtime, asset.path(), true); + assertEquals(200, head.status()); + assertEquals(content.length, head.contentLength()); + assertFalse(head.hasBody()); + MavenResponse get = support.serve(runtime, asset.path(), false); + try (var body = get.body()) { + assertArrayEquals(content, body.readAllBytes()); + } + } + + @Test + void treatsMissingDatabaseOrBlobStorageStateAsNotFound() { + when(assetDao.findAssetByPath(runtime.id(), "missing")).thenReturn(Optional.empty()); + assertThrows(MavenExceptions.MavenNotFoundException.class, + () -> support.bytes(runtime, "missing")); + + when(assetDao.findAssetByPath(runtime.id(), asset.path())).thenReturn(Optional.of(asset)); + when(assetDao.findBlobById(3L)).thenReturn(Optional.of(blob)); + when(storages.forBlobStoreId(blob.blobStoreId())).thenReturn(storage); + when(storage.stat(any())).thenReturn(Optional.empty()); + assertThrows(MavenExceptions.MavenNotFoundException.class, + () -> support.serve(runtime, asset.path(), false)); + } +} diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java new file mode 100644 index 00000000..4c22e6b9 --- /dev/null +++ b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java @@ -0,0 +1,385 @@ +package com.github.klboke.kkrepo.server.terraform; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.klboke.kkrepo.core.RepositoryFormat; +import com.github.klboke.kkrepo.core.RepositoryType; +import com.github.klboke.kkrepo.persistence.jdbc.api.TerraformRegistryDao; +import com.github.klboke.kkrepo.persistence.jdbc.api.model.AssetBlobRecord; +import com.github.klboke.kkrepo.persistence.jdbc.api.model.AssetRecord; +import com.github.klboke.kkrepo.protocol.terraform.TerraformPath; +import com.github.klboke.kkrepo.protocol.terraform.TerraformPathParser; +import com.github.klboke.kkrepo.server.maven.HttpRemoteFetcher; +import com.github.klboke.kkrepo.server.maven.MavenExceptions; +import com.github.klboke.kkrepo.server.maven.MavenResponse; +import com.github.klboke.kkrepo.server.maven.RepositoryRuntime; +import com.github.klboke.kkrepo.server.maven.RepositoryRuntimeRegistry; +import com.github.klboke.kkrepo.server.raw.RawProxyService; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +class TerraformServiceTest { + private static final String BASE = "https://repo.example/repository/terraform"; + private final TerraformPathParser paths = new TerraformPathParser(); + private ObjectMapper mapper; + private TerraformAssetSupport assets; + private TerraformArchiveInspector inspector; + private TerraformSigningService signing; + private TerraformSignatureVerifier signatureVerifier; + private TerraformRegistryDao registry; + private RepositoryRuntimeRegistry runtimes; + private RawProxyService proxy; + private HttpRemoteFetcher fetcher; + private TerraformService service; + + @BeforeEach + void setUp() { + mapper = new ObjectMapper(); + assets = mock(TerraformAssetSupport.class); + inspector = mock(TerraformArchiveInspector.class); + signing = mock(TerraformSigningService.class); + signatureVerifier = mock(TerraformSignatureVerifier.class); + registry = mock(TerraformRegistryDao.class); + runtimes = mock(RepositoryRuntimeRegistry.class); + proxy = mock(RawProxyService.class); + fetcher = mock(HttpRemoteFetcher.class); + service = new TerraformService( + mapper, assets, inspector, signing, signatureVerifier, registry, + new TerraformPublishLeaseManager(registry), runtimes, proxy, fetcher); + } + + @Test + void servesAndPublishesHostedModulesWithNexusUrlTokenSemantics() throws Exception { + RepositoryRuntime hosted = runtime(1, "terraform", RepositoryType.HOSTED, null, List.of()); + AssetRecord versionOne = asset(1, hosted, 11L, + "v1/modules/acme/network/aws/1.0.0/network.zip"); + AssetRecord versionTwo = asset(2, hosted, 12L, + "v1/modules/acme/network/aws/2.0.0/network.zip"); + when(assets.list(eq(hosted), anyString())).thenReturn(List.of(versionOne, versionTwo)); + when(assets.list(hosted, "v1/modules/acme/network/aws/2.0.0/")) + .thenReturn(List.of(versionTwo)); + + MavenResponse versions = service.get( + hosted, paths.parse("v1/modules/acme/network/aws/versions"), BASE, false); + Map versionsJson = json(versions); + assertTrue(versionsJson.toString().contains("2.0.0")); + assertTrue(versionsJson.toString().indexOf("2.0.0") < versionsJson.toString().indexOf("1.0.0")); + + MavenResponse download = service.get( + hosted, paths.parse("v1/modules/acme/network/aws/2.0.0/download"), BASE, + "dXNlcjpwYXNz", false); + assertEquals(204, download.status()); + assertEquals( + BASE + "/v1/modules/dXNlcjpwYXNz/acme/network/aws/2.0.0/network.zip", + download.headers().get("X-Terraform-Get")); + + when(assets.serve(hosted, versionTwo.path(), false)).thenReturn(MavenResponse.noBody(200)); + assertEquals(200, service.get(hosted, paths.parse(versionTwo.path()), BASE, false).status()); + assertThrows(MavenExceptions.MavenNotFoundException.class, + () -> service.get(hosted, paths.parse("unrelated/path"), BASE, false)); + + Path buffered = Files.createTempFile("terraform-module-service-test", ".zip"); + Files.writeString(buffered, "module"); + when(inspector.bufferAndInspect(any(), eq("new.zip"), eq(true), eq(null))).thenReturn(buffered); + when(assets.find(hosted, "v1/modules/acme/network/aws/3.0.0/new.zip")) + .thenReturn(Optional.empty(), Optional.of(asset(3, hosted, 13L, + "v1/modules/acme/network/aws/3.0.0/new.zip"))); + MavenResponse published = service.put( + hosted, + paths.parse("v1/modules/acme/network/aws/3.0.0/new.zip"), + new ByteArrayInputStream("module".getBytes(StandardCharsets.UTF_8)), + null, null, "alice", "127.0.0.1"); + assertEquals(201, published.status()); + assertFalse(Files.exists(buffered)); + verify(assets).store(eq(hosted), eq("v1/modules/acme/network/aws/3.0.0/new.zip"), + any(), eq("application/octet-stream"), any(), eq("alice"), eq("127.0.0.1")); + + assertThrows(MavenExceptions.WritePolicyDenied.class, + () -> service.put(runtime(2, "denied", RepositoryType.HOSTED, null, List.of(), "DENY"), + paths.parse("v1/modules/acme/network/aws/3.0.0/new.zip"), + new ByteArrayInputStream(new byte[0]), null, null, "alice", null)); + assertThrows(MavenExceptions.MethodNotAllowed.class, + () -> service.put(runtime(3, "proxy", RepositoryType.PROXY, "https://registry.example", List.of()), + paths.parse("v1/modules/acme/network/aws/3.0.0/new.zip"), + new ByteArrayInputStream(new byte[0]), null, null, "alice", null)); + assertThrows(MavenExceptions.MethodNotAllowed.class, + () -> service.put(hosted, paths.parse("v1/modules/acme/network/aws/versions"), + new ByteArrayInputStream(new byte[0]), null, null, "alice", null)); + } + + @Test + void servesHostedProviderMetadataArchivesAndSignatures() throws Exception { + RepositoryRuntime hosted = runtime(10, "terraform", RepositoryType.HOSTED, null, List.of()); + String archivePath = "v1/providers/acme/cloud/1.2.3/package/linux/" + + "terraform-provider-cloud_1.2.3_linux_amd64.zip"; + String sumsPath = "v1/providers/acme/cloud/1.2.3/metadata-r1/" + + "terraform-provider-cloud_1.2.3_SHA256SUMS"; + String signaturePath = sumsPath + ".sig"; + AssetRecord archive = asset(20, hosted, 21L, archivePath); + TerraformRegistryDao.ProviderPlatform platform = platform(hosted, archivePath, "abc123", 1); + TerraformRegistryDao.ProviderState state = new TerraformRegistryDao.ProviderState( + hosted.id(), "acme", "cloud", "1.2.3", 1, sumsPath, signaturePath, 2, Instant.now()); + when(assets.list(eq(hosted), anyString())).thenReturn(List.of(archive)); + when(registry.findProviderState(hosted.id(), "acme", "cloud", "1.2.3")) + .thenReturn(Optional.of(state)); + when(registry.listProviderPlatforms(hosted.id(), "acme", "cloud", "1.2.3")) + .thenReturn(List.of(platform)); + when(registry.findSigningKey(hosted.id(), 2)).thenReturn(Optional.of( + new TerraformRegistryDao.SigningKey(hosted.id(), 2, "0011223344556677", + "encrypted", "PUBLIC KEY", Instant.now()))); + + Map versions = json(service.get( + hosted, paths.parse("v1/providers/acme/cloud/versions"), BASE, false)); + assertTrue(versions.toString().contains("linux")); + assertTrue(versions.toString().contains("amd64")); + + Map download = json(service.get( + hosted, paths.parse("v1/providers/acme/cloud/1.2.3/download/linux/amd64"), BASE, + "token", false)); + assertEquals("abc123", download.get("shasum")); + assertTrue(download.get("download_url").toString().contains("/v1/providers/token/")); + assertTrue(download.toString().contains("0011223344556677")); + assertFalse(service.get( + hosted, paths.parse("v1/providers/acme/cloud/1.2.3/download/linux/amd64"), BASE, + null, true).hasBody()); + + when(assets.serve(eq(hosted), anyString(), anyBoolean())).thenReturn(MavenResponse.noBody(200)); + assertEquals(200, service.get(hosted, paths.parse(archivePath), BASE, false).status()); + assertEquals(200, service.get(hosted, paths.parse(sumsPath), BASE, true).status()); + assertEquals(200, service.get(hosted, paths.parse(signaturePath), BASE, false).status()); + + when(registry.listProviderPlatforms(hosted.id(), "acme", "cloud", "9.9.9")) + .thenReturn(List.of()); + assertThrows(MavenExceptions.MavenNotFoundException.class, + () -> service.get(hosted, + paths.parse("v1/providers/acme/cloud/9.9.9/package/linux/missing.zip"), BASE, false)); + } + + @Test + void publishesProviderPlatformAtomicallyUnderSharedLease() throws Exception { + RepositoryRuntime hosted = runtime(30, "terraform", RepositoryType.HOSTED, null, List.of()); + TerraformPath upload = paths.parse("v1/providers/acme/cloud/1.2.3/download/linux/amd64"); + String filename = "terraform-provider-cloud_1.2.3_linux_amd64.zip"; + String archivePath = "v1/providers/acme/cloud/1.2.3/package/linux/" + filename; + Path buffered = Files.createTempFile("terraform-provider-service-test", ".zip"); + Files.writeString(buffered, "provider"); + AssetRecord archive = asset(31, hosted, 32L, archivePath); + AssetBlobRecord blob = blob(32, "feedface", 8); + when(registry.tryAcquirePublishLease(anyString(), anyString(), any())).thenReturn(true); + when(registry.listProviderPlatforms(hosted.id(), "acme", "cloud", "1.2.3")) + .thenReturn(List.of()); + when(registry.findProviderState(hosted.id(), "acme", "cloud", "1.2.3")) + .thenReturn(Optional.empty()); + when(inspector.bufferAndInspect(any(), eq(filename), eq(false), eq("cloud"))).thenReturn(buffered); + when(assets.find(hosted, archivePath)).thenReturn(Optional.empty(), Optional.of(archive)); + when(assets.blob(archive)).thenReturn(blob); + when(signing.active(hosted)).thenReturn( + new TerraformSigningService.SigningMaterial(4, "AABBCCDDEEFF0011", "public", "private", "pass")); + when(signing.sign(any(), any())).thenReturn("signature".getBytes(StandardCharsets.UTF_8)); + + MavenResponse response = service.put( + hosted, upload, new ByteArrayInputStream("provider".getBytes(StandardCharsets.UTF_8)), + "application/zip", "attachment; filename=\"" + filename + "\"", "alice", "127.0.0.1"); + assertEquals(201, response.status()); + assertFalse(Files.exists(buffered)); + verify(registry).releasePublishLease(anyString(), anyString()); + ArgumentCaptor state = + ArgumentCaptor.forClass(TerraformRegistryDao.ProviderState.class); + verify(registry).publishProvider(any(TerraformRegistryDao.ProviderPlatform.class), state.capture()); + assertEquals(1, state.getValue().revision()); + assertEquals(4, state.getValue().signingKeyRevision()); + verify(assets, atLeastOnce()).storeBytes(eq(hosted), anyString(), any(), anyString(), any()); + + assertThrows(MavenExceptions.BadRequestException.class, + () -> service.put(hosted, upload, new ByteArrayInputStream(new byte[0]), null, + "attachment; filename=\"wrong.zip\"", "alice", null)); + } + + @Test + void rewritesAndVerifiesProxyProviderMetadataAndRoutes() throws Exception { + RepositoryRuntime proxyRuntime = runtime( + 40, "terraform-proxy", RepositoryType.PROXY, "https://registry.example/root", List.of()); + String filename = "terraform-provider-cloud_1.2.3_linux_amd64.zip"; + String checksum = "0123456789abcdef"; + Map provider = Map.of( + "protocols", List.of("5.0", "6.0"), + "os", "linux", + "arch", "amd64", + "filename", filename, + "download_url", "../../packages/" + filename, + "shasums_url", "../../packages/terraform-provider-cloud_1.2.3_SHA256SUMS", + "shasums_signature_url", "../../packages/terraform-provider-cloud_1.2.3_SHA256SUMS.sig", + "shasum", checksum, + "signing_keys", Map.of("gpg_public_keys", List.of(Map.of("ascii_armor", "PUBLIC")))); + byte[] sums = (checksum + " " + filename + "\n").getBytes(StandardCharsets.UTF_8); + when(proxy.getAssetFromUrl(eq(proxyRuntime), anyString(), anyString(), anyBoolean())) + .thenAnswer(invocation -> { + String local = invocation.getArgument(1); + String remote = invocation.getArgument(2); + if (local.equals(".terraform/upstream/discovery.json")) { + return response(mapper.writeValueAsBytes(Map.of( + "modules.v1", "/api/modules/", "providers.v1", "/api/providers/")), "application/json"); + } + if (remote.endsWith("/versions")) { + return response(mapper.writeValueAsBytes(Map.of( + "versions", List.of(Map.of("version", "1.2.3", "protocols", List.of("5.0"))))), + "application/json"); + } + if (remote.endsWith("/download/linux/amd64")) { + return response(mapper.writeValueAsBytes(provider), "application/json"); + } + if (remote.endsWith("_SHA256SUMS")) return response(sums, "text/plain"); + if (remote.endsWith("_SHA256SUMS.sig")) return response("sig".getBytes(), "application/octet-stream"); + return response("archive".getBytes(), "application/zip"); + }); + + Map versions = json(service.get( + proxyRuntime, paths.parse("v1/providers/acme/cloud/versions"), BASE, false)); + assertTrue(versions.toString().contains("1.2.3")); + + Map rewritten = json(service.get( + proxyRuntime, paths.parse("v1/providers/acme/cloud/1.2.3/download/linux/amd64"), + BASE, "url-token", false)); + assertTrue(rewritten.get("download_url").toString().contains("/v1/providers/url-token/")); + assertTrue(rewritten.get("shasums_url").toString().contains("metadata-proxy")); + verify(signatureVerifier).verify(eq(sums), any(), eq(List.of("PUBLIC"))); + verify(assets, atLeastOnce()).storeBytes(eq(proxyRuntime), anyString(), any(), + eq("application/json"), any()); + + String localArchive = "v1/providers/acme/cloud/1.2.3/package/linux/" + filename; + when(assets.bytes(eq(proxyRuntime), anyString())).thenReturn(mapper.writeValueAsBytes(Map.of( + "remoteUrl", "https://registry.example/packages/" + filename, "sha256", checksum))); + AssetRecord archive = asset(41, proxyRuntime, 42L, localArchive); + when(assets.find(proxyRuntime, localArchive)).thenReturn(Optional.of(archive)); + when(assets.blob(archive)).thenReturn(blob(42, checksum, 7)); + assertEquals(200, + service.get(proxyRuntime, paths.parse(localArchive), BASE, false).status()); + + when(assets.blob(archive)).thenReturn(blob(42, "different", 7)); + assertThrows(MavenExceptions.BadUpstreamException.class, + () -> service.get(proxyRuntime, paths.parse(localArchive), BASE, false)); + } + + @Test + void mergesGroupVersionsAndPersistsStableMemberBinding() throws Exception { + RepositoryRuntime first = runtime(51, "first", RepositoryType.HOSTED, null, List.of()); + RepositoryRuntime second = runtime(52, "second", RepositoryType.HOSTED, null, List.of()); + RepositoryRuntime group = runtime(50, "terraform", RepositoryType.GROUP, null, List.of(first, second)); + AssetRecord one = asset(51, first, 61L, "v1/modules/acme/network/aws/1.0.0/one.zip"); + AssetRecord two = asset(52, second, 62L, "v1/modules/acme/network/aws/2.0.0/two.zip"); + when(assets.list(eq(first), anyString())).thenReturn(List.of(one)); + when(assets.list(eq(second), anyString())).thenReturn(List.of(two)); + when(registry.findSourceBinding(eq(group.id()), anyString())).thenReturn(Optional.empty()); + + Map merged = json(service.get( + group, paths.parse("v1/modules/acme/network/aws/versions"), BASE, false)); + assertTrue(merged.toString().contains("1.0.0")); + assertTrue(merged.toString().contains("2.0.0")); + + MavenResponse download = service.get( + group, paths.parse("v1/modules/acme/network/aws/1.0.0/download"), BASE, false); + assertEquals(204, download.status()); + ArgumentCaptor binding = + ArgumentCaptor.forClass(TerraformRegistryDao.SourceBinding.class); + verify(registry).upsertSourceBinding(binding.capture()); + assertEquals(group.id(), binding.getValue().groupRepositoryId()); + assertEquals(first.id(), binding.getValue().memberRepositoryId()); + assertTrue(binding.getValue().bindingKey().contains("one.zip")); + + when(registry.findSourceBinding(group.id(), "asset:" + one.path())).thenReturn(Optional.of( + new TerraformRegistryDao.SourceBinding( + group.id(), "asset:" + one.path(), first.id(), 0, + Instant.now().plusSeconds(60), Instant.now()))); + when(runtimes.resolveById(first.id())).thenReturn(Optional.of(first)); + when(assets.serve(first, one.path(), false)).thenReturn(MavenResponse.noBody(200)); + assertEquals(200, service.get(group, paths.parse(one.path()), BASE, false).status()); + assertThrows(MavenExceptions.MavenNotFoundException.class, + () -> service.get(group, paths.parse(one.path()), BASE, true)); + } + + @Test + void rejectsMalformedProxyProviderMetadata() throws Exception { + RepositoryRuntime proxyRuntime = runtime( + 70, "terraform-proxy", RepositoryType.PROXY, "https://registry.example", List.of()); + when(proxy.getAssetFromUrl(eq(proxyRuntime), anyString(), anyString(), anyBoolean())) + .thenAnswer(invocation -> { + String local = invocation.getArgument(1); + if (local.equals(".terraform/upstream/discovery.json")) { + return response(mapper.writeValueAsBytes(Map.of("providers.v1", "/v1/providers/")), + "application/json"); + } + return response(mapper.writeValueAsBytes(Map.of("filename", "provider.zip")), + "application/json"); + }); + assertThrows(MavenExceptions.BadUpstreamException.class, + () -> service.get(proxyRuntime, + paths.parse("v1/providers/acme/cloud/1.2.3/download/linux/amd64"), BASE, false)); + verify(signatureVerifier, never()).verify(any(), any(), any()); + } + + private Map json(MavenResponse response) throws Exception { + try (var body = response.body()) { + return mapper.readValue(body, new TypeReference<>() {}); + } + } + + private static MavenResponse response(byte[] body, String contentType) { + return MavenResponse.ok(new ByteArrayInputStream(body), body.length, contentType, null, null); + } + + private static RepositoryRuntime runtime( + long id, String name, RepositoryType type, String remote, List members) { + return runtime(id, name, type, remote, members, "ALLOW_ONCE"); + } + + private static RepositoryRuntime runtime( + long id, String name, RepositoryType type, String remote, List members, + String writePolicy) { + return new RepositoryRuntime( + id, name, RepositoryFormat.TERRAFORM, type, "terraform-" + type.name().toLowerCase(), + true, 1L, writePolicy, null, null, true, remote, null, null, members); + } + + private static AssetRecord asset(long id, RepositoryRuntime runtime, Long blobId, String path) { + String name = path.substring(path.lastIndexOf('/') + 1); + return new AssetRecord( + id, runtime.id(), null, blobId, RepositoryFormat.TERRAFORM, path, new byte[32], name, + "terraform", "application/zip", 8L, null, Instant.now(), Map.of()); + } + + private static AssetBlobRecord blob(long id, String sha256, long size) { + return new AssetBlobRecord( + id, 1, "blob-ref", new byte[32], "object-key", new byte[32], "sha1", sha256, + "md5", size, "application/zip", "test", null, Instant.now(), Instant.now(), Map.of()); + } + + private static TerraformRegistryDao.ProviderPlatform platform( + RepositoryRuntime runtime, String path, String sha256, long revision) { + return new TerraformRegistryDao.ProviderPlatform( + runtime.id(), "acme", "cloud", "1.2.3", "linux", "amd64", + path.substring(path.lastIndexOf('/') + 1), path, sha256, "5.0", revision, Instant.now()); + } +} From 2871e48dfc299aebb99a2d29c9fff7da2b755c50 Mon Sep 17 00:00:00 2001 From: kl Date: Wed, 15 Jul 2026 08:35:35 +0800 Subject: [PATCH 04/53] test: expand Terraform migration and validation coverage --- .../TerraformArchiveInspectorTest.java | 119 ++++++++- ...formRepositoryDataMigrationWriterTest.java | 226 ++++++++++++++++++ .../TerraformSigningServiceTest.java | 12 + 3 files changed, 354 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformArchiveInspectorTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformArchiveInspectorTest.java index c2a2ebc1..b13c8220 100644 --- a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformArchiveInspectorTest.java +++ b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformArchiveInspectorTest.java @@ -7,13 +7,18 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import java.util.zip.GZIPOutputStream; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.archivers.tar.TarConstants; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; import org.junit.jupiter.api.Test; @@ -49,27 +54,135 @@ void rejectsArchiveWithBombLikeExpansionRatio() throws Exception { () -> inspector.bufferAndInspect(new ByteArrayInputStream(archive), "module.tar.gz", true, null)); } + @Test + void acceptsZipModulesAndProviders() throws Exception { + Path module = null; + Path provider = null; + try { + module = inspector.bufferAndInspect( + new ByteArrayInputStream(zipEntry( + "fixture/main.tf", "terraform {}".getBytes(StandardCharsets.UTF_8))), + "module.zip", true, null); + provider = inspector.bufferAndInspect( + new ByteArrayInputStream(zipEntry( + "terraform-provider-fixture_v1.2.3", "binary".getBytes(StandardCharsets.UTF_8))), + "provider.zip", false, "fixture"); + + assertTrue(Files.size(module) > 0); + assertTrue(Files.size(provider) > 0); + } finally { + if (module != null) Files.deleteIfExists(module); + if (provider != null) Files.deleteIfExists(provider); + } + } + + @Test + void rejectsArchivesWithoutExpectedContentAndUnsafeZipEntries() throws Exception { + byte[] noModule = zipEntry("fixture/README.md", "readme".getBytes(StandardCharsets.UTF_8)); + byte[] wrongProvider = zipEntry( + "terraform-provider-other", "binary".getBytes(StandardCharsets.UTF_8)); + byte[] traversal = zipEntry("../escape.tf", "terraform {}".getBytes(StandardCharsets.UTF_8)); + + assertThrows(MavenExceptions.BadRequestException.class, + () -> inspector.bufferAndInspect(new ByteArrayInputStream(noModule), "module.zip", true, null)); + assertThrows(MavenExceptions.BadRequestException.class, + () -> inspector.bufferAndInspect( + new ByteArrayInputStream(wrongProvider), "provider.zip", false, "fixture")); + assertThrows(MavenExceptions.BadRequestException.class, + () -> inspector.bufferAndInspect(new ByteArrayInputStream(traversal), "module.zip", true, null)); + } + + @Test + void rejectsDuplicateLinkAndDeviceEntries() throws Exception { + assertThrows(MavenExceptions.BadRequestException.class, + () -> inspector.bufferAndInspect( + new ByteArrayInputStream(tarEntries(List.of( + entry("fixture/main.tf", "terraform {}".getBytes(StandardCharsets.UTF_8)), + entry("fixture/main.tf", "terraform {}".getBytes(StandardCharsets.UTF_8))), ".tar.gz")), + "module.tar.gz", true, null)); + for (byte type : List.of(TarConstants.LF_SYMLINK, TarConstants.LF_FIFO)) { + assertThrows(MavenExceptions.BadRequestException.class, + () -> inspector.bufferAndInspect( + new ByteArrayInputStream(tarSpecialEntry("fixture/main.tf", type, ".tar.gz")), + "module.tar.gz", true, null)); + } + } + + @Test + void rejectsUnreadableUploadAndInvalidFilename() { + InputStream unreadable = new InputStream() { + @Override + public int read() throws IOException { + throw new IOException("broken stream"); + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + throw new IOException("broken stream"); + } + }; + assertThrows(MavenExceptions.BadRequestException.class, + () -> inspector.bufferAndInspect(unreadable, "module.zip", true, null)); + assertThrows(MavenExceptions.BadRequestException.class, + () -> inspector.bufferAndInspect(new ByteArrayInputStream(new byte[0]), null, true, null)); + } + private static byte[] tar(byte[] content, String suffix) throws IOException { return tarEntry("fixture/main.tf", content, suffix); } private static byte[] tarEntry(String name, byte[] content, String suffix) throws IOException { + return tarEntries(List.of(entry(name, content)), suffix); + } + + private static byte[] tarEntries(List entries, String suffix) throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (OutputStream compressed = compressor(bytes, suffix); + TarArchiveOutputStream tar = new TarArchiveOutputStream(compressed)) { + for (ArchiveEntry item : entries) { + TarArchiveEntry entry = new TarArchiveEntry(item.name()); + entry.setSize(item.content().length); + tar.putArchiveEntry(entry); + tar.write(item.content()); + tar.closeArchiveEntry(); + } + tar.finish(); + } + return bytes.toByteArray(); + } + + private static byte[] tarSpecialEntry(String name, byte type, String suffix) throws IOException { ByteArrayOutputStream bytes = new ByteArrayOutputStream(); try (OutputStream compressed = compressor(bytes, suffix); TarArchiveOutputStream tar = new TarArchiveOutputStream(compressed)) { - TarArchiveEntry entry = new TarArchiveEntry(name); - entry.setSize(content.length); + TarArchiveEntry entry = new TarArchiveEntry(name, type); + if (type == TarConstants.LF_SYMLINK) entry.setLinkName("fixture/target.tf"); tar.putArchiveEntry(entry); - tar.write(content); tar.closeArchiveEntry(); tar.finish(); } return bytes.toByteArray(); } + private static byte[] zipEntry(String name, byte[] content) throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ZipOutputStream zip = new ZipOutputStream(bytes)) { + zip.putNextEntry(new ZipEntry(name)); + zip.write(content); + zip.closeEntry(); + } + return bytes.toByteArray(); + } + + private static ArchiveEntry entry(String name, byte[] content) { + return new ArchiveEntry(name, content); + } + private static OutputStream compressor(OutputStream out, String suffix) throws IOException { if (suffix.endsWith("gz") || suffix.endsWith("tgz")) return new GZIPOutputStream(out); if (suffix.endsWith("xz") || suffix.endsWith("txz")) return new XZCompressorOutputStream(out); return new BZip2CompressorOutputStream(out); } + + private record ArchiveEntry(String name, byte[] content) {} } diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformRepositoryDataMigrationWriterTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformRepositoryDataMigrationWriterTest.java index 52735edd..3b9e1978 100644 --- a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformRepositoryDataMigrationWriterTest.java +++ b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformRepositoryDataMigrationWriterTest.java @@ -1,9 +1,36 @@ package com.github.klboke.kkrepo.server.terraform; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import com.github.klboke.kkrepo.core.RepositoryFormat; +import com.github.klboke.kkrepo.core.RepositoryType; +import com.github.klboke.kkrepo.persistence.jdbc.api.TerraformRegistryDao; +import com.github.klboke.kkrepo.persistence.jdbc.api.model.AssetBlobRecord; +import com.github.klboke.kkrepo.persistence.jdbc.api.model.AssetRecord; +import com.github.klboke.kkrepo.persistence.jdbc.api.model.RepositoryDataMigrationAssetRecord; +import com.github.klboke.kkrepo.persistence.jdbc.api.model.RepositoryRecord; +import com.github.klboke.kkrepo.protocol.terraform.TerraformPath; +import com.github.klboke.kkrepo.server.maven.MavenResponse; +import com.github.klboke.kkrepo.server.maven.RepositoryRuntime; +import com.github.klboke.kkrepo.server.maven.RepositoryRuntimeRegistry; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; class TerraformRepositoryDataMigrationWriterTest { @Test @@ -19,5 +46,204 @@ void selectsOnlyNexusModuleAndProviderArchivesForMigration() { "/v1/providers/kkrepo/fixture/versions.json")); assertFalse(TerraformRepositoryDataMigrationWriter.isMigratableTerraformPath( "/v1/providers/kkrepo/fixture/1.2.3/download/linux/amd64/../../escape.zip")); + assertFalse(TerraformRepositoryDataMigrationWriter.isMigratableTerraformPath(null)); } + + @Test + void reusesAlreadyMigratedModuleAndValidatesItsBlob() { + Fixture fixture = fixture(); + String path = "v1/modules/kkrepo/network/aws/1.2.3/module.zip"; + RepositoryDataMigrationAssetRecord source = source(path, 7L, "application/zip", "nexus", "10.0.0.1"); + AssetRecord asset = asset(fixture.runtime(), 101, 102L, path, 103L); + AssetBlobRecord blob = blob(102, 7, "module-object"); + when(fixture.assets().find(fixture.runtime(), path)).thenReturn(Optional.of(asset)); + when(fixture.assets().blob(asset)).thenReturn(blob); + + TerraformRepositoryDataMigrationWriter.MigratedAsset migrated = fixture.writer().write( + fixture.repository(), source, new ByteArrayInputStream(new byte[0]), null, true); + + assertEquals(103L, migrated.componentId()); + assertEquals(101L, migrated.assetId()); + assertEquals(102L, migrated.assetBlobId()); + assertEquals("module-object", migrated.assetBlobObjectKey()); + verify(fixture.service(), never()).put(any(), any(), any(), any(), any(), any(), any()); + + RepositoryDataMigrationAssetRecord wrongSize = source(path, 8L, null, null, null); + assertThrows(IllegalStateException.class, + () -> fixture.writer().write( + fixture.repository(), wrongSize, new ByteArrayInputStream(new byte[0]), null, true)); + when(fixture.assets().blob(asset)).thenReturn(null); + assertThrows(IllegalStateException.class, + () -> fixture.writer().write( + fixture.repository(), source, new ByteArrayInputStream(new byte[0]), null, false)); + } + + @Test + void reusesPublishedProviderPlatformOnlyAfterSharedStateConfirmsPublication() { + Fixture fixture = fixture(); + String sourcePath = "v1/providers/kkrepo/fixture/1.2.3/download/linux/amd64/" + + "terraform-provider-fixture_1.2.3_linux_amd64.zip"; + String targetPath = "v1/providers/kkrepo/fixture/1.2.3/package/linux/" + + "terraform-provider-fixture_1.2.3_linux_amd64.zip"; + RepositoryDataMigrationAssetRecord source = source(sourcePath, 9L, "application/zip", null, null); + AssetRecord asset = asset(fixture.runtime(), 201, 202L, targetPath, null); + when(fixture.assets().find(fixture.runtime(), targetPath)).thenReturn(Optional.of(asset)); + when(fixture.registry().listProviderPlatforms( + fixture.runtime().id(), "kkrepo", "fixture", "1.2.3")).thenReturn(List.of( + new TerraformRegistryDao.ProviderPlatform( + fixture.runtime().id(), "kkrepo", "fixture", "1.2.3", "linux", "amd64", + "terraform-provider-fixture_1.2.3_linux_amd64.zip", targetPath, "sha256", "5.0", + 1, Instant.now()))); + when(fixture.assets().blob(asset)).thenReturn(blob(202, 9, "provider-object")); + + TerraformRepositoryDataMigrationWriter.MigratedAsset migrated = fixture.writer().write( + fixture.repository(), source, new ByteArrayInputStream(new byte[0]), null, true); + + assertEquals(201L, migrated.assetId()); + assertEquals("provider-object", migrated.assetBlobObjectKey()); + verify(fixture.service(), never()).put(any(), any(), any(), any(), any(), any(), any()); + } + + @Test + void buffersAndReplaysModuleThroughNormalPublicationPath() { + Fixture fixture = fixture(); + String path = "v1/modules/kkrepo/network/aws/1.2.3/module.zip"; + byte[] content = "module!".getBytes(StandardCharsets.UTF_8); + RepositoryDataMigrationAssetRecord source = source( + "/" + path + "/", (long) content.length, "application/zip", "nexus-user", "10.0.0.2"); + AssetRecord stored = asset(fixture.runtime(), 301, 302L, path, 303L); + when(fixture.assets().find(fixture.runtime(), path)) + .thenReturn(Optional.empty(), Optional.of(stored)); + when(fixture.assets().blob(stored)).thenReturn(blob(302, content.length, "module-new")); + when(fixture.service().put(any(), any(), any(), any(), any(), any(), any())) + .thenReturn(MavenResponse.created()); + + TerraformRepositoryDataMigrationWriter.MigratedAsset migrated = fixture.writer().write( + fixture.repository(), source, new ByteArrayInputStream(content), "application/x-terraform-module", true); + + assertEquals(301L, migrated.assetId()); + ArgumentCaptor pathCaptor = ArgumentCaptor.forClass(TerraformPath.class); + verify(fixture.service()).put( + eq(fixture.runtime()), pathCaptor.capture(), any(InputStream.class), + eq("application/x-terraform-module"), eq(null), eq("nexus-user"), eq("10.0.0.2")); + assertEquals(TerraformPath.Kind.MODULE_ARCHIVE, pathCaptor.getValue().kind()); + assertEquals(path, pathCaptor.getValue().rawPath()); + } + + @Test + void mapsNexusProviderArchiveToCanonicalPackagePathAndDefaultsMetadata() { + Fixture fixture = fixture(); + String filename = "terraform-provider-fixture_1.2.3_linux_amd64.zip"; + String sourcePath = "/v1/providers/kkrepo/fixture/1.2.3/download/linux/amd64/" + filename; + String targetPath = "v1/providers/kkrepo/fixture/1.2.3/package/linux/" + filename; + byte[] content = "provider".getBytes(StandardCharsets.UTF_8); + RepositoryDataMigrationAssetRecord source = source(sourcePath, (long) content.length, null, null, null); + AssetRecord stored = asset(fixture.runtime(), 401, 402L, targetPath, null); + when(fixture.assets().find(fixture.runtime(), targetPath)) + .thenReturn(Optional.empty(), Optional.of(stored)); + when(fixture.registry().listProviderPlatforms( + fixture.runtime().id(), "kkrepo", "fixture", "1.2.3")).thenReturn(List.of()); + when(fixture.assets().blob(stored)).thenReturn(blob(402, content.length, "provider-new")); + when(fixture.service().put(any(), any(), any(), any(), any(), any(), any())) + .thenReturn(MavenResponse.created()); + + fixture.writer().write( + fixture.repository(), source, new ByteArrayInputStream(content), " ", true); + + ArgumentCaptor pathCaptor = ArgumentCaptor.forClass(TerraformPath.class); + verify(fixture.service()).put( + eq(fixture.runtime()), pathCaptor.capture(), any(InputStream.class), + eq("application/octet-stream"), eq("attachment; filename=\"" + filename + "\""), + eq("nexus-migration"), eq(null)); + assertEquals(TerraformPath.Kind.PROVIDER_DOWNLOAD, pathCaptor.getValue().kind()); + assertEquals("linux", pathCaptor.getValue().os()); + assertEquals("amd64", pathCaptor.getValue().arch()); + } + + @Test + void rejectsInvalidTargetsSizeMismatchAndMissingPublishedAsset() { + Fixture fixture = fixture(); + byte[] content = "module".getBytes(StandardCharsets.UTF_8); + String modulePath = "v1/modules/kkrepo/network/aws/1.2.3/module.zip"; + + RepositoryRecord raw = new RepositoryRecord( + 1L, "raw", RepositoryFormat.RAW, RepositoryType.HOSTED, "raw-hosted", true, + 1L, null, null, null, null, "ALLOW_ONCE", true, Map.of()); + assertThrows(IllegalArgumentException.class, + () -> fixture.writer().write(raw, source(modulePath, 6L, null, null, null), + new ByteArrayInputStream(content), null, true)); + + when(fixture.runtimes().resolveById(fixture.repository().id())).thenReturn(Optional.empty()); + assertThrows(IllegalArgumentException.class, + () -> fixture.writer().write(fixture.repository(), source(modulePath, 6L, null, null, null), + new ByteArrayInputStream(content), null, true)); + + when(fixture.runtimes().resolveById(fixture.repository().id())) + .thenReturn(Optional.of(fixture.runtime())); + assertThrows(IllegalArgumentException.class, + () -> fixture.writer().write(fixture.repository(), source("v1/providers/invalid", 1L, null, null, null), + new ByteArrayInputStream(content), null, true)); + + when(fixture.assets().find(fixture.runtime(), modulePath)).thenReturn(Optional.empty()); + assertThrows(IllegalStateException.class, + () -> fixture.writer().write(fixture.repository(), source(modulePath, 99L, null, null, null), + new ByteArrayInputStream(content), null, true)); + verify(fixture.service(), never()).put(any(), any(), any(), any(), any(), any(), any()); + + when(fixture.service().put(any(), any(), any(), any(), any(), any(), any())) + .thenReturn(MavenResponse.created()); + assertThrows(IllegalStateException.class, + () -> fixture.writer().write(fixture.repository(), source(modulePath, 6L, null, null, null), + new ByteArrayInputStream(content), null, false)); + } + + private static Fixture fixture() { + TerraformService service = mock(TerraformService.class); + TerraformAssetSupport assets = mock(TerraformAssetSupport.class); + TerraformRegistryDao registry = mock(TerraformRegistryDao.class); + RepositoryRuntimeRegistry runtimes = mock(RepositoryRuntimeRegistry.class); + RepositoryRuntime runtime = new RepositoryRuntime( + 7L, "terraform-hosted", RepositoryFormat.TERRAFORM, RepositoryType.HOSTED, + "terraform-hosted", true, 1L, "ALLOW_ONCE", null, null, true, + null, null, null, List.of()); + RepositoryRecord repository = new RepositoryRecord( + runtime.id(), runtime.name(), RepositoryFormat.TERRAFORM, RepositoryType.HOSTED, + "terraform-hosted", true, 1L, null, null, null, null, "ALLOW_ONCE", true, Map.of()); + when(runtimes.resolveById(runtime.id())).thenReturn(Optional.of(runtime)); + return new Fixture( + new TerraformRepositoryDataMigrationWriter(service, assets, registry, runtimes), + service, assets, registry, runtimes, runtime, repository); + } + + private static RepositoryDataMigrationAssetRecord source( + String path, Long size, String contentType, String createdBy, String createdByIp) { + return new RepositoryDataMigrationAssetRecord( + 1L, 2L, "asset-1", "component-1", path, new byte[32], RepositoryFormat.TERRAFORM, + "kkrepo", "fixture", "1.2.3", "archive", contentType, size, "blob-ref", + Instant.now(), null, Instant.now(), Instant.now(), createdBy, createdByIp, + "PENDING", 0, null, null, null, null, null, null, Map.of(), Instant.now()); + } + + private static AssetRecord asset( + RepositoryRuntime runtime, long id, Long blobId, String path, Long componentId) { + return new AssetRecord( + id, runtime.id(), componentId, blobId, RepositoryFormat.TERRAFORM, path, new byte[32], + path.substring(path.lastIndexOf('/') + 1), "terraform", "application/zip", 7L, + null, Instant.now(), Map.of()); + } + + private static AssetBlobRecord blob(long id, long size, String objectKey) { + return new AssetBlobRecord( + id, 1L, "blob-ref", new byte[32], objectKey, new byte[32], "sha1", "sha256", "md5", + size, "application/zip", "nexus", null, Instant.now(), Instant.now(), Map.of()); + } + + private record Fixture( + TerraformRepositoryDataMigrationWriter writer, + TerraformService service, + TerraformAssetSupport assets, + TerraformRegistryDao registry, + RepositoryRuntimeRegistry runtimes, + RepositoryRuntime runtime, + RepositoryRecord repository) {} } diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformSigningServiceTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformSigningServiceTest.java index 9de20ca1..6a558acb 100644 --- a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformSigningServiceTest.java +++ b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformSigningServiceTest.java @@ -1,6 +1,7 @@ package com.github.klboke.kkrepo.server.terraform; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; @@ -11,6 +12,7 @@ import com.github.klboke.kkrepo.core.RepositoryFormat; import com.github.klboke.kkrepo.core.RepositoryType; import com.github.klboke.kkrepo.persistence.jdbc.api.TerraformRegistryDao; +import com.github.klboke.kkrepo.server.maven.MavenExceptions; import com.github.klboke.kkrepo.server.maven.RepositoryRuntime; import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; @@ -62,5 +64,15 @@ void generatedDetachedSignatureVerifiesWithPublishedPublicKey() throws Exception primary); signature.update(contents); assertTrue(signature.verify()); + + TerraformSignatureVerifier verifier = new TerraformSignatureVerifier(); + assertThrows(MavenExceptions.BadUpstreamException.class, + () -> verifier.verify("tampered".getBytes(StandardCharsets.UTF_8), detached, + List.of(material.publicArmor()))); + assertThrows(MavenExceptions.BadUpstreamException.class, + () -> verifier.verify(contents, detached, List.of())); + assertThrows(MavenExceptions.BadUpstreamException.class, + () -> verifier.verify(contents, "not a signature".getBytes(StandardCharsets.UTF_8), + List.of(material.publicArmor()))); } } From e96bba7e4ab637d426516af401c844330446c550 Mon Sep 17 00:00:00 2001 From: kl Date: Wed, 15 Jul 2026 08:52:10 +0800 Subject: [PATCH 05/53] fix: address Terraform review findings --- .../protocol/terraform/TerraformPathParser.java | 4 +++- .../terraform/TerraformPathParserTest.java | 7 +++++++ .../server/terraform/TerraformService.java | 1 + .../server/terraform/TerraformServiceTest.java | 16 ++++++++++++++++ 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/protocol-terraform/src/main/java/com/github/klboke/kkrepo/protocol/terraform/TerraformPathParser.java b/protocol-terraform/src/main/java/com/github/klboke/kkrepo/protocol/terraform/TerraformPathParser.java index 0e03af83..e1626d1a 100644 --- a/protocol-terraform/src/main/java/com/github/klboke/kkrepo/protocol/terraform/TerraformPathParser.java +++ b/protocol-terraform/src/main/java/com/github/klboke/kkrepo/protocol/terraform/TerraformPathParser.java @@ -174,7 +174,9 @@ public static void requireFilename(String value) { private static String decodeOnce(String value) { try { - return URLDecoder.decode(value, StandardCharsets.UTF_8); + // URLDecoder implements form semantics, where a literal '+' means a space. This value is a + // path segment, so protect literal plus signs while still decoding percent-encoded bytes. + return URLDecoder.decode(value.replace("+", "%2B"), StandardCharsets.UTF_8); } catch (IllegalArgumentException e) { throw new IllegalArgumentException("Invalid Terraform URL token encoding", e); } diff --git a/protocol-terraform/src/test/java/com/github/klboke/kkrepo/protocol/terraform/TerraformPathParserTest.java b/protocol-terraform/src/test/java/com/github/klboke/kkrepo/protocol/terraform/TerraformPathParserTest.java index 94b0ffa7..271ddf3b 100644 --- a/protocol-terraform/src/test/java/com/github/klboke/kkrepo/protocol/terraform/TerraformPathParserTest.java +++ b/protocol-terraform/src/test/java/com/github/klboke/kkrepo/protocol/terraform/TerraformPathParserTest.java @@ -36,6 +36,13 @@ void stripsOneUrlTokenSegmentBeforeProtocolDispatch() { "v1/providers/generic-token/acme/cloud/1.2.3/download/linux/amd64"); assertEquals("generic-token", provider.credentialSegment()); assertEquals(TerraformPath.Kind.PROVIDER_DOWNLOAD, provider.path().kind()); + + var literalPlus = parser.parseRequestPath( + "v1/modules/dXNlcjpwYXNz+/acme/network/aws/versions"); + assertEquals("dXNlcjpwYXNz+", literalPlus.credentialSegment()); + var encodedPlus = parser.parseRequestPath( + "v1/providers/generic%2Btoken/acme/cloud/1.2.3/download/linux/amd64"); + assertEquals("generic+token", encodedPlus.credentialSegment()); } @Test diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java index 4c47720c..85304a0f 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java @@ -245,6 +245,7 @@ private MavenResponse putModule( private MavenResponse putProvider( RepositoryRuntime runtime, TerraformPath path, InputStream body, String contentType, String contentDisposition, String actor, String ip) { + enforceWrite(runtime, path.rawPath()); String filename = filename(contentDisposition); String expectedPrefix = "terraform-provider-" + path.name() + "_" + path.version() + "_" + path.os() + "_" + path.arch(); diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java index 4c22e6b9..1ea33c61 100644 --- a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java +++ b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java @@ -218,6 +218,22 @@ hosted, upload, new ByteArrayInputStream("provider".getBytes(StandardCharsets.UT "attachment; filename=\"wrong.zip\"", "alice", null)); } + @Test + void deniesProviderPublishBeforeAcquiringTheSharedLease() { + RepositoryRuntime denied = runtime( + 31, "terraform-denied", RepositoryType.HOSTED, null, List.of(), "DENY"); + TerraformPath upload = paths.parse("v1/providers/acme/cloud/1.2.3/download/linux/amd64"); + String filename = "terraform-provider-cloud_1.2.3_linux_amd64.zip"; + + assertThrows(MavenExceptions.WritePolicyDenied.class, + () -> service.put( + denied, upload, new ByteArrayInputStream(new byte[0]), "application/zip", + "attachment; filename=\"" + filename + "\"", "alice", "127.0.0.1")); + + verify(registry, never()).tryAcquirePublishLease(anyString(), anyString(), any()); + verify(inspector, never()).bufferAndInspect(any(), anyString(), anyBoolean(), anyString()); + } + @Test void rewritesAndVerifiesProxyProviderMetadataAndRoutes() throws Exception { RepositoryRuntime proxyRuntime = runtime( From 18dd871e43f4ab7d9c84d0ae27e3796d329e784d Mon Sep 17 00:00:00 2001 From: kl Date: Wed, 15 Jul 2026 09:10:20 +0800 Subject: [PATCH 06/53] fix: tighten Terraform authorization and group metadata --- .../security/RepositorySecurityFilter.java | 25 +++++- .../server/terraform/TerraformService.java | 30 ++++++- .../RepositorySecurityFilterTest.java | 86 +++++++++++++++++++ .../terraform/TerraformServiceTest.java | 49 +++++++++++ 4 files changed, 188 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/security/RepositorySecurityFilter.java b/server/src/main/java/com/github/klboke/kkrepo/server/security/RepositorySecurityFilter.java index 70919052..5987e306 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/security/RepositorySecurityFilter.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/security/RepositorySecurityFilter.java @@ -6,10 +6,12 @@ import com.github.klboke.kkrepo.auth.RepositoryPermission; import com.github.klboke.kkrepo.core.RepositoryFormat; import com.github.klboke.kkrepo.core.RepositoryType; +import com.github.klboke.kkrepo.persistence.jdbc.api.AssetDao; import com.github.klboke.kkrepo.persistence.jdbc.api.RepositoryDao; import com.github.klboke.kkrepo.persistence.jdbc.api.model.RepositoryRecord; import com.github.klboke.kkrepo.protocol.pub.PubPath; import com.github.klboke.kkrepo.protocol.pub.PubPathParser; +import com.github.klboke.kkrepo.protocol.terraform.TerraformPath; import com.github.klboke.kkrepo.protocol.terraform.TerraformPathParser; import com.github.klboke.kkrepo.server.npm.NpmTokenService; import jakarta.servlet.FilterChain; @@ -42,6 +44,7 @@ public class RepositorySecurityFilter extends OncePerRequestFilter { private final SecurityAuthenticationService authenticationService; private final AccessDecisionService accessDecisionService; private final RepositoryDao repositoryDao; + private final AssetDao assetDao; private final ForwardedHeaderPolicy forwardedHeaderPolicy; private final NexusLegacyUiCompatibility legacyUi; @@ -49,11 +52,13 @@ public RepositorySecurityFilter( SecurityAuthenticationService authenticationService, AccessDecisionService accessDecisionService, RepositoryDao repositoryDao, + AssetDao assetDao, ForwardedHeaderPolicy forwardedHeaderPolicy, NexusLegacyUiCompatibility legacyUi) { this.authenticationService = authenticationService; this.accessDecisionService = accessDecisionService; this.repositoryDao = repositoryDao; + this.assetDao = assetDao; this.forwardedHeaderPolicy = forwardedHeaderPolicy; this.legacyUi = legacyUi; } @@ -135,7 +140,7 @@ private AccessDecision decide( RepositoryRecord repository, RepositoryRequest target) { AccessDecision lastDenied = AccessDecision.deny("missing permission"); - for (PermissionAction action : target.actions(repository.format())) { + for (PermissionAction action : actionsForDecision(repository, target)) { AccessDecision decision = accessDecisionService.decide( subject.permissionSubject(), new RepositoryPermission(repository.name(), repository.format(), target.path(), action)); @@ -147,6 +152,24 @@ private AccessDecision decide( return lastDenied; } + private List actionsForDecision( + RepositoryRecord repository, RepositoryRequest target) { + if (repository.format() != RepositoryFormat.TERRAFORM + || !"PUT".equalsIgnoreCase(target.method())) { + return target.actions(repository.format()); + } + TerraformPath path = TERRAFORM_PATH_PARSER.parse(target.path()); + if (path.kind() == TerraformPath.Kind.MODULE_ARCHIVE) { + return assetDao.findAssetByPath(repository.id(), target.path()).isPresent() + ? List.of(PermissionAction.EDIT) + : List.of(PermissionAction.ADD); + } + if (path.kind() == TerraformPath.Kind.PROVIDER_DOWNLOAD) { + return List.of(PermissionAction.ADD); + } + return List.of(PermissionAction.EDIT); + } + private Optional resolve(HttpServletRequest request) { String method = request.getMethod(); String uri = stripContextPath(request); diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java index 85304a0f..f21460e7 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java @@ -485,7 +485,13 @@ private MavenResponse mergeGroupVersions( } } else { for (Map version : (List>) body.getOrDefault("versions", List.of())) { - versions.putIfAbsent(string(version.get("version")), version); + String number = string(version.get("version")); + Map existing = versions.get(number); + if (existing == null) { + versions.put(number, new LinkedHashMap<>(version)); + } else { + mergeProviderVersion(existing, version); + } } } } catch (RuntimeException ignored) { @@ -500,6 +506,28 @@ private MavenResponse mergeGroupVersions( : json(Map.of("versions", sorted), headOnly); } + @SuppressWarnings("unchecked") + private static void mergeProviderVersion( + Map existing, Map incoming) { + Map> platforms = new LinkedHashMap<>(); + for (Map source : List.of(existing, incoming)) { + for (Map platform + : (List>) source.getOrDefault("platforms", List.of())) { + String key = string(platform.get("os")) + "\u0000" + string(platform.get("arch")); + platforms.putIfAbsent(key, platform); + } + } + existing.put("platforms", List.copyOf(platforms.values())); + + Set protocols = new LinkedHashSet<>(); + for (Map source : List.of(existing, incoming)) { + for (Object protocol : (List) source.getOrDefault("protocols", List.of())) { + if (protocol != null) protocols.add(protocol.toString()); + } + } + existing.put("protocols", List.copyOf(protocols)); + } + @SuppressWarnings("unchecked") private MavenResponse bindGroupResponse( RepositoryRuntime group, RepositoryRuntime member, TerraformPath request, MavenResponse response) { diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/security/RepositorySecurityFilterTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/security/RepositorySecurityFilterTest.java index a2b0fd14..16e738e2 100644 --- a/server/src/test/java/com/github/klboke/kkrepo/server/security/RepositorySecurityFilterTest.java +++ b/server/src/test/java/com/github/klboke/kkrepo/server/security/RepositorySecurityFilterTest.java @@ -13,7 +13,9 @@ import com.github.klboke.kkrepo.core.RepositoryType; import com.github.klboke.kkrepo.persistence.jdbc.api.RepositoryDao; import com.github.klboke.kkrepo.persistence.jdbc.api.SecurityDao; +import com.github.klboke.kkrepo.persistence.jdbc.api.model.AssetRecord; import com.github.klboke.kkrepo.persistence.jdbc.api.model.RepositoryRecord; +import com.github.klboke.kkrepo.server.support.dao.AssetDaoAdapter; import com.github.klboke.kkrepo.server.support.dao.RepositoryDaoAdapter; import com.github.klboke.kkrepo.server.support.dao.SecurityDaoAdapter; import jakarta.servlet.DispatcherType; @@ -21,6 +23,7 @@ import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; import java.lang.reflect.Proxy; +import java.time.Instant; import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; @@ -448,6 +451,47 @@ void repositoryContentRoutesUseNexusBreadActionMapping() throws Exception { assertRepositoryContentAction("DELETE", PermissionAction.DELETE); } + @Test + void terraformModuleCreatesRequireAddAndOverwritesRequireEdit() throws Exception { + String path = "v1/modules/acme/network/aws/1.2.3/module.zip"; + assertTerraformModuleAction(path, Set.of(), PermissionAction.ADD); + assertTerraformModuleAction(path, Set.of(path), PermissionAction.EDIT); + } + + @Test + void terraformModuleOverwriteDoesNotFallBackToAddWhenEditIsDenied() throws Exception { + String path = "v1/modules/acme/network/aws/1.2.3/module.zip"; + StubAuthenticationService authentication = + new StubAuthenticationService(Optional.of(subject("alice"))); + RecordingDecisionService decisions = new RecordingDecisionService(AccessDecision.deny("missing edit")) { + @Override + public AccessDecision decide(PermissionSubject subject, RepositoryPermission permission) { + super.decide(subject, permission); + return permission.action() == PermissionAction.ADD + ? AccessDecision.allow() + : AccessDecision.deny("missing edit"); + } + }; + RepositorySecurityFilter filter = new RepositorySecurityFilter( + authentication, + decisions, + new FakeRepositoryDao(repository( + "terraform-hosted", RepositoryFormat.TERRAFORM, RepositoryType.HOSTED)), + new FakeAssetDao(Set.of(path)), + new ForwardedHeaderPolicy(""), + new NexusLegacyUiCompatibility(false)); + ResponseState response = new ResponseState(); + ChainState chain = new ChainState(); + + filter.doFilter( + request("PUT", "/repository/terraform-hosted/" + path), response.proxy(), chain); + + assertEquals(0, chain.calls); + assertEquals(1, decisions.decisions); + assertEquals(PermissionAction.EDIT, decisions.permission.action()); + assertEquals(HttpServletResponse.SC_FORBIDDEN, response.status); + } + @Test void cargoPublishRouteRequiresAddPermission() throws Exception { assertCargoRouteAction("PUT", "/repository/cargo-hosted/api/v1/crates/new", PermissionAction.ADD); @@ -666,6 +710,7 @@ private static RepositorySecurityFilter filter( authenticationService, accessDecisionService, repositoryDao, + new FakeAssetDao(Set.of()), forwardedHeaderPolicy, new NexusLegacyUiCompatibility(false)); } @@ -683,6 +728,7 @@ private static RepositorySecurityFilter filter( authenticationService, accessDecisionService, repositoryDao, + new FakeAssetDao(Set.of()), new ForwardedHeaderPolicy(""), new NexusLegacyUiCompatibility(legacyUiEnabled)); } @@ -695,6 +741,29 @@ private static void assertRepositoryContentAction(String method, PermissionActio action); } + private static void assertTerraformModuleAction( + String path, Set existingPaths, PermissionAction expected) throws Exception { + StubAuthenticationService authentication = + new StubAuthenticationService(Optional.of(subject("alice"))); + RecordingDecisionService decisions = new RecordingDecisionService(AccessDecision.allow()); + RepositorySecurityFilter filter = new RepositorySecurityFilter( + authentication, + decisions, + new FakeRepositoryDao(repository( + "terraform-hosted", RepositoryFormat.TERRAFORM, RepositoryType.HOSTED)), + new FakeAssetDao(existingPaths), + new ForwardedHeaderPolicy(""), + new NexusLegacyUiCompatibility(false)); + ResponseState response = new ResponseState(); + ChainState chain = new ChainState(); + + filter.doFilter( + request("PUT", "/repository/terraform-hosted/" + path), response.proxy(), chain); + + assertEquals(1, chain.calls); + assertEquals(expected, decisions.permission.action()); + } + private static void assertRepositoryPathAction( RepositoryRecord repository, String method, @@ -994,6 +1063,23 @@ public Optional findByName(String name) { } } + private static final class FakeAssetDao extends AssetDaoAdapter { + private final Set existingPaths; + + private FakeAssetDao(Set existingPaths) { + this.existingPaths = existingPaths; + } + + @Override + public Optional findAssetByPath(long repositoryId, String path) { + if (!existingPaths.contains(path)) return Optional.empty(); + return Optional.of(new AssetRecord( + 1L, repositoryId, null, null, RepositoryFormat.TERRAFORM, path, new byte[32], + path.substring(path.lastIndexOf('/') + 1), "terraform", "application/zip", 1L, + null, Instant.EPOCH, Map.of())); + } + } + private static class ResponseState { private int status; private String message; diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java index 1ea33c61..60a2df00 100644 --- a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java +++ b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java @@ -37,6 +37,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; @@ -336,6 +337,46 @@ void mergesGroupVersionsAndPersistsStableMemberBinding() throws Exception { () -> service.get(group, paths.parse(one.path()), BASE, true)); } + @Test + @SuppressWarnings("unchecked") + void mergesPlatformsForTheSameProviderVersionAcrossGroupMembers() throws Exception { + RepositoryRuntime first = runtime(53, "first-provider", RepositoryType.HOSTED, null, List.of()); + RepositoryRuntime second = runtime(54, "second-provider", RepositoryType.HOSTED, null, List.of()); + RepositoryRuntime group = runtime( + 55, "terraform-group", RepositoryType.GROUP, null, List.of(first, second)); + String linuxPath = "v1/providers/acme/cloud/1.2.3/package/linux/" + + "terraform-provider-cloud_1.2.3_linux_amd64.zip"; + String darwinPath = "v1/providers/acme/cloud/1.2.3/package/darwin/" + + "terraform-provider-cloud_1.2.3_darwin_arm64.zip"; + when(assets.list(first, "v1/providers/acme/cloud/")) + .thenReturn(List.of(asset(53, first, 63L, linuxPath))); + when(assets.list(second, "v1/providers/acme/cloud/")) + .thenReturn(List.of(asset(54, second, 64L, darwinPath))); + TerraformRegistryDao.ProviderState firstState = new TerraformRegistryDao.ProviderState( + first.id(), "acme", "cloud", "1.2.3", 1, "first-sums", "first-sig", 1, Instant.now()); + TerraformRegistryDao.ProviderState secondState = new TerraformRegistryDao.ProviderState( + second.id(), "acme", "cloud", "1.2.3", 1, "second-sums", "second-sig", 1, Instant.now()); + when(registry.findProviderState(first.id(), "acme", "cloud", "1.2.3")) + .thenReturn(Optional.of(firstState)); + when(registry.findProviderState(second.id(), "acme", "cloud", "1.2.3")) + .thenReturn(Optional.of(secondState)); + when(registry.listProviderPlatforms(first.id(), "acme", "cloud", "1.2.3")) + .thenReturn(List.of(providerPlatform(first, linuxPath, "linux", "amd64"))); + when(registry.listProviderPlatforms(second.id(), "acme", "cloud", "1.2.3")) + .thenReturn(List.of(providerPlatform(second, darwinPath, "darwin", "arm64"))); + + Map merged = json(service.get( + group, paths.parse("v1/providers/acme/cloud/versions"), BASE, false)); + List> versions = (List>) merged.get("versions"); + List> platforms = + (List>) versions.getFirst().get("platforms"); + + assertEquals(1, versions.size()); + assertEquals(Set.of("linux/amd64", "darwin/arm64"), Set.copyOf(platforms.stream() + .map(platform -> platform.get("os") + "/" + platform.get("arch")) + .toList())); + } + @Test void rejectsMalformedProxyProviderMetadata() throws Exception { RepositoryRuntime proxyRuntime = runtime( @@ -398,4 +439,12 @@ private static TerraformRegistryDao.ProviderPlatform platform( runtime.id(), "acme", "cloud", "1.2.3", "linux", "amd64", path.substring(path.lastIndexOf('/') + 1), path, sha256, "5.0", revision, Instant.now()); } + + private static TerraformRegistryDao.ProviderPlatform providerPlatform( + RepositoryRuntime runtime, String path, String os, String arch) { + return new TerraformRegistryDao.ProviderPlatform( + runtime.id(), "acme", "cloud", "1.2.3", os, arch, + path.substring(path.lastIndexOf('/') + 1), path, "sha256-" + os, "5.0", 1, + Instant.now()); + } } From df60e7f178ea3e4e286458d3da18bac3fcad8819 Mon Sep 17 00:00:00 2001 From: kl Date: Wed, 15 Jul 2026 09:32:01 +0800 Subject: [PATCH 07/53] fix: enforce Terraform module coordinates and metadata ttl --- .../kkrepo/server/raw/RawProxyService.java | 24 ++++++++++--- .../security/RepositorySecurityFilter.java | 8 ++++- .../server/terraform/TerraformService.java | 36 +++++++++++++++---- .../server/raw/RawProxyServiceTest.java | 27 ++++++++++++-- .../RepositorySecurityFilterTest.java | 17 +++++---- .../terraform/TerraformServiceTest.java | 25 ++++++++++--- 6 files changed, 114 insertions(+), 23 deletions(-) diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/raw/RawProxyService.java b/server/src/main/java/com/github/klboke/kkrepo/server/raw/RawProxyService.java index 224aff6e..ce989a8e 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/raw/RawProxyService.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/raw/RawProxyService.java @@ -100,9 +100,24 @@ public MavenResponse getAsset(RepositoryRuntime runtime, String path, String rem } public MavenResponse getAssetFromUrl(RepositoryRuntime runtime, String path, String remoteUrl, boolean headOnly) { + return getAssetFromUrl( + runtime, path, remoteUrl, runtime.contentMaxAgeMinutesOrDefault(), + HttpRemoteFetcher.TimeoutProfile.CONTENT, headOnly); + } + + public MavenResponse getMetadataFromUrl( + RepositoryRuntime runtime, String path, String remoteUrl, boolean headOnly) { + return getAssetFromUrl( + runtime, path, remoteUrl, runtime.metadataMaxAgeMinutesOrDefault(), + HttpRemoteFetcher.TimeoutProfile.METADATA, headOnly); + } + + private MavenResponse getAssetFromUrl( + RepositoryRuntime runtime, String path, String remoteUrl, int maxAgeMinutes, + HttpRemoteFetcher.TimeoutProfile timeoutProfile, boolean headOnly) { Optional cached = lookupCached(runtime, path); Instant now = Instant.now(); - if (cached.isPresent() && isFresh(cached.get(), runtime.contentMaxAgeMinutesOrDefault(), now)) { + if (cached.isPresent() && isFresh(cached.get(), maxAgeMinutes, now)) { return reader.serveSnapshot(cached.get(), headOnly, path, runtime.rawContentDispositionOrDefault()); } if (negativeCache.isNotFoundCached(runtime, path)) { @@ -114,7 +129,7 @@ public MavenResponse getAssetFromUrl(RepositoryRuntime runtime, String path, Str } throw new MavenExceptions.BadUpstreamException("Upstream temporarily blocked: " + remoteUrl); } - return fetchAndCacheUrl(runtime, path, remoteUrl, cached, headOnly, now); + return fetchAndCacheUrl(runtime, path, remoteUrl, cached, headOnly, now, timeoutProfile); } private Optional lookupCached(RepositoryRuntime runtime, String path) { @@ -150,7 +165,8 @@ private MavenResponse fetchAndCacheUrl( String remoteUrl, Optional cached, boolean headOnly, - Instant now) { + Instant now, + HttpRemoteFetcher.TimeoutProfile timeoutProfile) { String etag = null; Instant lastModified = null; if (cached.isPresent() && cached.get().blob() != null) { @@ -159,7 +175,7 @@ private MavenResponse fetchAndCacheUrl( lastModified = instantAttr(attrs, "remoteLastModified"); } HttpRemoteFetcher.Request req = cachePopulationRequest( - runtime, remoteUrl, etag, lastModified); + runtime, remoteUrl, etag, lastModified).withTimeoutProfile(timeoutProfile); return fetchAndCache(runtime, path, cached, headOnly, now, req); } diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/security/RepositorySecurityFilter.java b/server/src/main/java/com/github/klboke/kkrepo/server/security/RepositorySecurityFilter.java index 5987e306..14894119 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/security/RepositorySecurityFilter.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/security/RepositorySecurityFilter.java @@ -160,7 +160,13 @@ private List actionsForDecision( } TerraformPath path = TERRAFORM_PATH_PARSER.parse(target.path()); if (path.kind() == TerraformPath.Kind.MODULE_ARCHIVE) { - return assetDao.findAssetByPath(repository.id(), target.path()).isPresent() + String prefix = "v1/modules/" + path.namespace() + "/" + path.name() + "/" + + path.system() + "/" + path.version() + "/"; + boolean exists = assetDao.listAssetsByPrefix(repository.id(), prefix).stream() + .filter(asset -> asset.path().startsWith(prefix)) + .anyMatch(asset -> TERRAFORM_PATH_PARSER.parse(asset.path()).kind() + == TerraformPath.Kind.MODULE_ARCHIVE); + return exists ? List.of(PermissionAction.EDIT) : List.of(PermissionAction.ADD); } diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java index f21460e7..6c7c2e16 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java @@ -226,7 +226,7 @@ private MavenResponse servePublishedProviderMetadata( private MavenResponse putModule( RepositoryRuntime runtime, TerraformPath path, InputStream body, String contentType, String actor, String ip) { - enforceWrite(runtime, path.rawPath()); + enforceModuleWrite(runtime, path); Path buffered = inspector.bufferAndInspect(body, path.filename(), true, null); try (InputStream in = Files.newInputStream(buffered)) { assets.store(runtime, path.rawPath(), in, contentType == null ? OCTET : contentType, @@ -322,7 +322,7 @@ private MavenResponse proxyGet( private MavenResponse proxyMetadata(RepositoryRuntime runtime, TerraformPath path, boolean headOnly) { String remote = remoteUrl(runtime, path); String cachePath = ".terraform/upstream/" + sha256(remote) + ".json"; - MavenResponse cached = proxy.getAssetFromUrl(runtime, cachePath, remote, false); + MavenResponse cached = proxy.getMetadataFromUrl(runtime, cachePath, remote, false); byte[] bytes = responseBytes(cached); return bytes(bytes, JSON, headOnly); } @@ -359,7 +359,8 @@ private MavenResponse proxyProviderDownload( RepositoryRuntime runtime, TerraformPath path, RequestUrls urls, boolean headOnly) { String remote = remoteUrl(runtime, path); String cachePath = ".terraform/upstream/" + sha256(remote) + ".json"; - Map body = readJson(responseBytes(proxy.getAssetFromUrl(runtime, cachePath, remote, false))); + Map body = readJson(responseBytes( + proxy.getMetadataFromUrl(runtime, cachePath, remote, false))); String filename = string(body.get("filename")); TerraformPathParser.requireFilename(filename); String download = absolute(remote, string(body.get("download_url"))); @@ -373,8 +374,10 @@ private MavenResponse proxyProviderDownload( + "/metadata-proxy/" + sumsFile; String localSignature = localSums + ".sig"; String expected = string(body.get("shasum")); - byte[] shasumsBytes = responseBytes(proxy.getAssetFromUrl(runtime, localSums, sums, false)); - byte[] signatureBytes = responseBytes(proxy.getAssetFromUrl(runtime, localSignature, signature, false)); + byte[] shasumsBytes = responseBytes( + proxy.getMetadataFromUrl(runtime, localSums, sums, false)); + byte[] signatureBytes = responseBytes( + proxy.getMetadataFromUrl(runtime, localSignature, signature, false)); verifyChecksumEntry(shasumsBytes, expected, filename); signatureVerifier.verify(shasumsBytes, signatureBytes, upstreamPublicKeys(body)); storeRoute(runtime, localDownload, download, expected); @@ -573,7 +576,8 @@ private String discovery(RepositoryRuntime runtime, String key) { String root = ensureSlash(runtime.proxyRemoteUrl()); String remote = root + ".well-known/terraform.json"; String local = ".terraform/upstream/discovery.json"; - Map document = readJson(responseBytes(proxy.getAssetFromUrl(runtime, local, remote, false))); + Map document = readJson(responseBytes( + proxy.getMetadataFromUrl(runtime, local, remote, false))); String value = string(document.get(key)); if (value == null || value.isBlank()) { throw new MavenExceptions.BadUpstreamException("Terraform discovery omitted " + key); @@ -601,6 +605,26 @@ private void enforceWrite(RepositoryRuntime runtime, String path) { } } + private void enforceModuleWrite(RepositoryRuntime runtime, TerraformPath path) { + String policy = runtime.writePolicy() == null + ? "ALLOW_ONCE" : runtime.writePolicy().toUpperCase(Locale.ROOT); + if ("DENY".equals(policy)) { + throw new MavenExceptions.WritePolicyDenied("Repository write policy is DENY"); + } + String prefix = "v1/modules/" + path.namespace() + "/" + path.name() + "/" + + path.system() + "/" + path.version() + "/"; + List existing = assets.list(runtime, prefix).stream() + .filter(asset -> asset.path().startsWith(prefix)) + .filter(asset -> paths.parse(asset.path()).kind() == TerraformPath.Kind.MODULE_ARCHIVE) + .toList(); + if (existing.isEmpty()) return; + if ("ALLOW".equals(policy) && existing.size() == 1 + && existing.getFirst().path().equals(path.rawPath())) { + return; + } + throw new MavenExceptions.WritePolicyDenied("Terraform module version already exists"); + } + private String filename(String contentDisposition) { return contentDispositionFilename(contentDisposition); } diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/raw/RawProxyServiceTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/raw/RawProxyServiceTest.java index 56b4b975..cd5c8b13 100644 --- a/server/src/test/java/com/github/klboke/kkrepo/server/raw/RawProxyServiceTest.java +++ b/server/src/test/java/com/github/klboke/kkrepo/server/raw/RawProxyServiceTest.java @@ -114,6 +114,24 @@ void servesStaleCacheWhileUpstreamIsBlocked() { assertSame(expected, fixture.service.getAsset(runtime, "file.txt", true)); } + @Test + void metadataRequestsUseTheMetadataMaxAgeInsteadOfTheContentMaxAge() { + Fixture fixture = fixture(); + RepositoryRuntime runtime = runtime(RepositoryType.PROXY, 60, 1); + CachedAssetMetadata cached = snapshot(Instant.now().minusSeconds(5 * 60)); + MavenResponse expected = MavenResponse.noBody(200); + when(fixture.cache.find(eq(runtime.id()), eq("versions.json"), any())) + .thenReturn(Optional.of(cached)); + when(fixture.proxyStateDao.isBlocked(eq(runtime.id()), any())).thenReturn(true); + when(fixture.reader.serveSnapshot(cached, false, "versions.json", "ATTACHMENT")) + .thenReturn(expected); + + assertSame(expected, fixture.service.getMetadataFromUrl( + runtime, "versions.json", "https://upstream.example.test/versions", false)); + + verify(fixture.proxyStateDao).isBlocked(eq(runtime.id()), any()); + } + @Test void reportsBlockedUpstreamWhenNoCacheExists() { Fixture fixture = fixture(); @@ -154,6 +172,11 @@ private static Fixture fixture() { } private static RepositoryRuntime runtime(RepositoryType type, int maxAgeMinutes) { + return runtime(type, maxAgeMinutes, maxAgeMinutes); + } + + private static RepositoryRuntime runtime( + RepositoryType type, int contentMaxAgeMinutes, int metadataMaxAgeMinutes) { return new RepositoryRuntime( 10L, "raw-proxy", @@ -167,8 +190,8 @@ private static RepositoryRuntime runtime(RepositoryType type, int maxAgeMinutes) null, true, "https://upstream.example.test/", - maxAgeMinutes, - maxAgeMinutes, + contentMaxAgeMinutes, + metadataMaxAgeMinutes, true, "ATTACHMENT", List.of()); diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/security/RepositorySecurityFilterTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/security/RepositorySecurityFilterTest.java index 16e738e2..be1608a1 100644 --- a/server/src/test/java/com/github/klboke/kkrepo/server/security/RepositorySecurityFilterTest.java +++ b/server/src/test/java/com/github/klboke/kkrepo/server/security/RepositorySecurityFilterTest.java @@ -25,6 +25,7 @@ import java.lang.reflect.Proxy; import java.time.Instant; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -456,6 +457,8 @@ void terraformModuleCreatesRequireAddAndOverwritesRequireEdit() throws Exception String path = "v1/modules/acme/network/aws/1.2.3/module.zip"; assertTerraformModuleAction(path, Set.of(), PermissionAction.ADD); assertTerraformModuleAction(path, Set.of(path), PermissionAction.EDIT); + assertTerraformModuleAction( + "v1/modules/acme/network/aws/1.2.3/alternate.zip", Set.of(path), PermissionAction.EDIT); } @Test @@ -1071,12 +1074,14 @@ private FakeAssetDao(Set existingPaths) { } @Override - public Optional findAssetByPath(long repositoryId, String path) { - if (!existingPaths.contains(path)) return Optional.empty(); - return Optional.of(new AssetRecord( - 1L, repositoryId, null, null, RepositoryFormat.TERRAFORM, path, new byte[32], - path.substring(path.lastIndexOf('/') + 1), "terraform", "application/zip", 1L, - null, Instant.EPOCH, Map.of())); + public List listAssetsByPrefix(long repositoryId, String prefix) { + return existingPaths.stream() + .filter(path -> path.startsWith(prefix)) + .map(path -> new AssetRecord( + 1L, repositoryId, null, null, RepositoryFormat.TERRAFORM, path, new byte[32], + path.substring(path.lastIndexOf('/') + 1), "terraform", "application/zip", 1L, + null, Instant.EPOCH, Map.of())) + .toList(); } } diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java index 60a2df00..e2941cf4 100644 --- a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java +++ b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java @@ -122,6 +122,19 @@ void servesAndPublishesHostedModulesWithNexusUrlTokenSemantics() throws Exceptio () -> service.put(runtime(2, "denied", RepositoryType.HOSTED, null, List.of(), "DENY"), paths.parse("v1/modules/acme/network/aws/3.0.0/new.zip"), new ByteArrayInputStream(new byte[0]), null, null, "alice", null)); + assertThrows(MavenExceptions.WritePolicyDenied.class, + () -> service.put(hosted, + paths.parse("v1/modules/acme/network/aws/1.0.0/alternate.zip"), + new ByteArrayInputStream(new byte[0]), null, null, "alice", null)); + RepositoryRuntime allow = runtime( + 4, "allow", RepositoryType.HOSTED, null, List.of(), "ALLOW"); + when(assets.list(allow, "v1/modules/acme/network/aws/1.0.0/")) + .thenReturn(List.of(asset( + 4, allow, 14L, "v1/modules/acme/network/aws/1.0.0/network.zip"))); + assertThrows(MavenExceptions.WritePolicyDenied.class, + () -> service.put(allow, + paths.parse("v1/modules/acme/network/aws/1.0.0/alternate.zip"), + new ByteArrayInputStream(new byte[0]), null, null, "alice", null)); assertThrows(MavenExceptions.MethodNotAllowed.class, () -> service.put(runtime(3, "proxy", RepositoryType.PROXY, "https://registry.example", List.of()), paths.parse("v1/modules/acme/network/aws/3.0.0/new.zip"), @@ -252,7 +265,7 @@ void rewritesAndVerifiesProxyProviderMetadataAndRoutes() throws Exception { "shasum", checksum, "signing_keys", Map.of("gpg_public_keys", List.of(Map.of("ascii_armor", "PUBLIC")))); byte[] sums = (checksum + " " + filename + "\n").getBytes(StandardCharsets.UTF_8); - when(proxy.getAssetFromUrl(eq(proxyRuntime), anyString(), anyString(), anyBoolean())) + when(proxy.getMetadataFromUrl(eq(proxyRuntime), anyString(), anyString(), anyBoolean())) .thenAnswer(invocation -> { String local = invocation.getArgument(1); String remote = invocation.getArgument(2); @@ -269,9 +282,13 @@ void rewritesAndVerifiesProxyProviderMetadataAndRoutes() throws Exception { return response(mapper.writeValueAsBytes(provider), "application/json"); } if (remote.endsWith("_SHA256SUMS")) return response(sums, "text/plain"); - if (remote.endsWith("_SHA256SUMS.sig")) return response("sig".getBytes(), "application/octet-stream"); - return response("archive".getBytes(), "application/zip"); + if (remote.endsWith("_SHA256SUMS.sig")) { + return response("sig".getBytes(StandardCharsets.UTF_8), "application/octet-stream"); + } + return response("archive".getBytes(StandardCharsets.UTF_8), "application/zip"); }); + when(proxy.getAssetFromUrl(eq(proxyRuntime), anyString(), anyString(), anyBoolean())) + .thenReturn(MavenResponse.noBody(200)); Map versions = json(service.get( proxyRuntime, paths.parse("v1/providers/acme/cloud/versions"), BASE, false)); @@ -381,7 +398,7 @@ void mergesPlatformsForTheSameProviderVersionAcrossGroupMembers() throws Excepti void rejectsMalformedProxyProviderMetadata() throws Exception { RepositoryRuntime proxyRuntime = runtime( 70, "terraform-proxy", RepositoryType.PROXY, "https://registry.example", List.of()); - when(proxy.getAssetFromUrl(eq(proxyRuntime), anyString(), anyString(), anyBoolean())) + when(proxy.getMetadataFromUrl(eq(proxyRuntime), anyString(), anyString(), anyBoolean())) .thenAnswer(invocation -> { String local = invocation.getArgument(1); if (local.equals(".terraform/upstream/discovery.json")) { From 9c74c5e01f9a5e81102439c43860f3e56d33699e Mon Sep 17 00:00:00 2001 From: kl Date: Wed, 15 Jul 2026 09:53:56 +0800 Subject: [PATCH 08/53] fix: preserve Terraform go-getter module sources --- .../server/terraform/TerraformService.java | 20 ++++++- .../terraform/TerraformServiceTest.java | 55 +++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java index 6c7c2e16..9f3ee65a 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java @@ -341,7 +341,10 @@ private MavenResponse proxyModuleDownload(RepositoryRuntime runtime, TerraformPa if (upstream == null || upstream.isBlank()) { throw new MavenExceptions.BadUpstreamException("Terraform upstream omitted X-Terraform-Get"); } - String absolute = URI.create(remote).resolve(upstream).toString(); + String absolute = httpModuleSource(remote, upstream); + if (absolute == null) { + return MavenResponse.noBody(204).withHeader("X-Terraform-Get", upstream); + } String filename = safeRemoteFilename(absolute, path.name() + "_" + path.version() + ".zip"); String local = "v1/modules/" + path.namespace() + "/" + path.name() + "/" + path.system() @@ -748,6 +751,21 @@ private static String absolute(String base, String value) { return URI.create(base).resolve(value).toString(); } + private static String httpModuleSource(String base, String value) { + String candidate = value; + if (value.startsWith("/") || value.startsWith("./") || value.startsWith("../")) { + try { + candidate = URI.create(base).resolve(value).toString(); + } catch (RuntimeException e) { + throw new MavenExceptions.BadUpstreamException("Terraform upstream module URL is invalid"); + } + } + return candidate.regionMatches(true, 0, "http://", 0, "http://".length()) + || candidate.regionMatches(true, 0, "https://", 0, "https://".length()) + ? candidate + : null; + } + private static String repositoryPath(String url, String repository) { if (url == null) return null; String marker = "/repository/" + repository + "/"; diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java index e2941cf4..b1984266 100644 --- a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java +++ b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java @@ -8,6 +8,7 @@ import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -30,6 +31,7 @@ import com.github.klboke.kkrepo.server.maven.RepositoryRuntimeRegistry; import com.github.klboke.kkrepo.server.raw.RawProxyService; import java.io.ByteArrayInputStream; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -317,6 +319,50 @@ void rewritesAndVerifiesProxyProviderMetadataAndRoutes() throws Exception { () -> service.get(proxyRuntime, paths.parse(localArchive), BASE, false)); } + @Test + void preservesGoGetterSourceFromProxyModuleDownload() throws Exception { + RepositoryRuntime proxyRuntime = runtime( + 42, "terraform-proxy", RepositoryType.PROXY, "https://registry.example/root", List.of()); + when(proxy.getMetadataFromUrl(eq(proxyRuntime), anyString(), anyString(), anyBoolean())) + .thenReturn(response(mapper.writeValueAsBytes(Map.of("modules.v1", "/api/modules/")), + "application/json")); + String source = "git::https://github.com/acme/network.git?ref=v1.2.3"; + respond(fetcher, new HttpRemoteFetcher.Result( + 204, Map.of("X-Terraform-Get", source), new ByteArrayInputStream(new byte[0]))); + + MavenResponse download = service.get( + proxyRuntime, paths.parse("v1/modules/acme/network/aws/1.2.3/download"), BASE, false); + + assertEquals(204, download.status()); + assertEquals(source, download.headers().get("X-Terraform-Get")); + verify(assets, never()).storeBytes(eq(proxyRuntime), anyString(), any(), anyString(), any()); + } + + @Test + void resolvesAndCachesRelativeHttpSourceFromProxyModuleDownload() throws Exception { + RepositoryRuntime proxyRuntime = runtime( + 43, "terraform-proxy", RepositoryType.PROXY, "https://registry.example/root", List.of()); + when(proxy.getMetadataFromUrl(eq(proxyRuntime), anyString(), anyString(), anyBoolean())) + .thenReturn(response(mapper.writeValueAsBytes(Map.of("modules.v1", "/api/modules/")), + "application/json")); + respond(fetcher, new HttpRemoteFetcher.Result( + 204, Map.of("X-Terraform-Get", "../packages/network.zip"), + new ByteArrayInputStream(new byte[0]))); + + MavenResponse download = service.get( + proxyRuntime, paths.parse("v1/modules/acme/network/aws/1.2.3/download"), BASE, false); + + assertEquals(204, download.status()); + assertEquals(BASE + "/v1/modules/acme/network/aws/1.2.3/network.zip", + download.headers().get("X-Terraform-Get")); + ArgumentCaptor route = ArgumentCaptor.forClass(byte[].class); + verify(assets).storeBytes(eq(proxyRuntime), anyString(), route.capture(), + eq("application/json"), any()); + assertEquals( + "https://registry.example/api/modules/acme/network/aws/packages/network.zip", + mapper.readValue(route.getValue(), new TypeReference>() {}).get("remoteUrl")); + } + @Test void mergesGroupVersionsAndPersistsStableMemberBinding() throws Exception { RepositoryRuntime first = runtime(51, "first", RepositoryType.HOSTED, null, List.of()); @@ -424,6 +470,15 @@ private static MavenResponse response(byte[] body, String contentType) { return MavenResponse.ok(new ByteArrayInputStream(body), body.length, contentType, null, null); } + private static void respond(HttpRemoteFetcher fetcher, HttpRemoteFetcher.Result result) + throws IOException { + doAnswer(invocation -> { + @SuppressWarnings("unchecked") + HttpRemoteFetcher.ResultHandler handler = invocation.getArgument(2); + return handler.handle(result); + }).when(fetcher).fetchWithBodyRetry(any(), anyString(), any()); + } + private static RepositoryRuntime runtime( long id, String name, RepositoryType type, String remote, List members) { return runtime(id, name, type, remote, members, "ALLOW_ONCE"); From 0c7ee3ac432a6cf1199558a5d2801bb8f3330ebb Mon Sep 17 00:00:00 2001 From: kl Date: Wed, 15 Jul 2026 10:16:01 +0800 Subject: [PATCH 09/53] fix: bound Terraform group keys and parse URL tokens --- .../terraform/TerraformPathParser.java | 21 ++++++++++++++++--- .../terraform/TerraformPathParserTest.java | 6 ++++++ .../server/terraform/TerraformService.java | 8 +++++-- .../terraform/TerraformServiceTest.java | 7 ++++--- 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/protocol-terraform/src/main/java/com/github/klboke/kkrepo/protocol/terraform/TerraformPathParser.java b/protocol-terraform/src/main/java/com/github/klboke/kkrepo/protocol/terraform/TerraformPathParser.java index e1626d1a..969312eb 100644 --- a/protocol-terraform/src/main/java/com/github/klboke/kkrepo/protocol/terraform/TerraformPathParser.java +++ b/protocol-terraform/src/main/java/com/github/klboke/kkrepo/protocol/terraform/TerraformPathParser.java @@ -39,13 +39,11 @@ public ParsedRequest parseRequestPath(String rawPath) { directFailure = e; } if (direct.kind() != TerraformPath.Kind.UNKNOWN) return new ParsedRequest(raw, null, direct); - List segments = splitAndValidate(raw); + List segments = splitRequestSegments(raw); if (segments.size() <= 2 || !"v1".equals(segments.get(0)) || !("modules".equals(segments.get(1)) || "providers".equals(segments.get(1)))) { return new ParsedRequest(raw, null, direct); } - String credential = decodeOnce(segments.get(2)); - if (credential.length() > 4096) throw new IllegalArgumentException("Terraform URL token is too long"); List canonical = new ArrayList<>(segments); canonical.remove(2); String normalized = String.join("/", canonical); @@ -60,6 +58,8 @@ public ParsedRequest parseRequestPath(String rawPath) { if (directFailure != null) throw directFailure; return new ParsedRequest(raw, null, direct); } + String credential = decodeOnce(segments.get(2)); + if (credential.length() > 4096) throw new IllegalArgumentException("Terraform URL token is too long"); return new ParsedRequest(normalized, credential, parsed); } @@ -149,6 +149,21 @@ private static List splitAndValidate(String raw) { return List.copyOf(result); } + private static List splitRequestSegments(String raw) { + if (raw.indexOf('\0') >= 0 || raw.toLowerCase(Locale.ROOT).contains("%00")) { + throw new IllegalArgumentException("Unsafe Terraform repository path"); + } + String[] values = raw.split("/", -1); + List result = new ArrayList<>(values.length); + for (String value : values) { + if (value.isEmpty() || ".".equals(value) || "..".equals(value) || value.indexOf('\\') >= 0) { + throw new IllegalArgumentException("Unsafe Terraform repository path segment"); + } + result.add(value); + } + return List.copyOf(result); + } + private static String normalize(String rawPath) { String value = rawPath == null ? "" : rawPath.trim(); while (value.startsWith("/")) value = value.substring(1); diff --git a/protocol-terraform/src/test/java/com/github/klboke/kkrepo/protocol/terraform/TerraformPathParserTest.java b/protocol-terraform/src/test/java/com/github/klboke/kkrepo/protocol/terraform/TerraformPathParserTest.java index 271ddf3b..9ec353e9 100644 --- a/protocol-terraform/src/test/java/com/github/klboke/kkrepo/protocol/terraform/TerraformPathParserTest.java +++ b/protocol-terraform/src/test/java/com/github/klboke/kkrepo/protocol/terraform/TerraformPathParserTest.java @@ -43,6 +43,10 @@ void stripsOneUrlTokenSegmentBeforeProtocolDispatch() { var encodedPlus = parser.parseRequestPath( "v1/providers/generic%2Btoken/acme/cloud/1.2.3/download/linux/amd64"); assertEquals("generic+token", encodedPlus.credentialSegment()); + var encodedSlash = parser.parseRequestPath( + "v1/modules/dTo%2F/acme/network/aws/versions"); + assertEquals("dTo/", encodedSlash.credentialSegment()); + assertEquals("v1/modules/acme/network/aws/versions", encodedSlash.canonicalPath()); } @Test @@ -63,6 +67,8 @@ void rejectsTraversalEncodingAndNonSemanticVersions() { "v1/modules/acme/network/aws/latest/download")) { assertThrows(IllegalArgumentException.class, () -> parser.parse(path), path); } + assertThrows(IllegalArgumentException.class, + () -> parser.parseRequestPath("v1/modules/token/acme/%2Fetc/aws/versions")); } @Test diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java index 9f3ee65a..27598a33 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java @@ -455,7 +455,7 @@ private MavenResponse groupGet( || path.kind() == TerraformPath.Kind.PROVIDER_VERSIONS) { return mergeGroupVersions(group, path, urls, headOnly); } - String bindingKey = "asset:" + path.rawPath(); + String bindingKey = sourceBindingKey(path.rawPath()); Optional existing = registry.findSourceBinding(group.id(), bindingKey); if (existing.isPresent()) { RepositoryRuntime member = runtimes.resolveById(existing.get().memberRepositoryId()).orElse(null); @@ -555,7 +555,7 @@ private MavenResponse bindGroupResponse( for (String bound : pathsToBind) { if (bound == null || bound.isBlank()) continue; registry.upsertSourceBinding(new TerraformRegistryDao.SourceBinding( - group.id(), "asset:" + bound, member.id(), memberRevision(member, request), + group.id(), sourceBindingKey(bound), member.id(), memberRevision(member, request), now.plus(24, ChronoUnit.HOURS), now)); } return response; @@ -816,6 +816,10 @@ private static String sha256(String value) { } } + private static String sourceBindingKey(String assetPath) { + return "asset:sha256:" + sha256(assetPath); + } + private static MavenExceptions.MavenNotFoundException notFound(String path) { return new MavenExceptions.MavenNotFoundException(path); } diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java index b1984266..18afb84e 100644 --- a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java +++ b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java @@ -387,11 +387,12 @@ void mergesGroupVersionsAndPersistsStableMemberBinding() throws Exception { verify(registry).upsertSourceBinding(binding.capture()); assertEquals(group.id(), binding.getValue().groupRepositoryId()); assertEquals(first.id(), binding.getValue().memberRepositoryId()); - assertTrue(binding.getValue().bindingKey().contains("one.zip")); + assertTrue(binding.getValue().bindingKey().startsWith("asset:sha256:")); + assertEquals(77, binding.getValue().bindingKey().length()); - when(registry.findSourceBinding(group.id(), "asset:" + one.path())).thenReturn(Optional.of( + when(registry.findSourceBinding(group.id(), binding.getValue().bindingKey())).thenReturn(Optional.of( new TerraformRegistryDao.SourceBinding( - group.id(), "asset:" + one.path(), first.id(), 0, + group.id(), binding.getValue().bindingKey(), first.id(), 0, Instant.now().plusSeconds(60), Instant.now()))); when(runtimes.resolveById(first.id())).thenReturn(Optional.of(first)); when(assets.serve(first, one.path(), false)).thenReturn(MavenResponse.noBody(200)); From 579303d613a472f12490c3d5183b7b4031af37c9 Mon Sep 17 00:00:00 2001 From: kl Date: Wed, 15 Jul 2026 10:38:50 +0800 Subject: [PATCH 10/53] fix: coordinate Terraform publication and metadata history --- .../server/terraform/TerraformService.java | 49 +++++++++++++++---- .../upload/ComponentUploadErrorAdvice.java | 3 +- .../terraform/TerraformServiceTest.java | 38 +++++++++++++- .../ComponentUploadErrorAdviceTest.java | 10 ++++ 4 files changed, 88 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java index 27598a33..d57790f7 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java @@ -219,22 +219,48 @@ private MavenResponse servePublishedProviderMetadata( .orElseThrow(() -> notFound(path.rawPath())); String published = path.kind() == TerraformPath.Kind.PROVIDER_SHA256SUMS ? state.shasumsPath() : state.signaturePath(); - if (!published.equals(path.rawPath())) throw notFound(path.rawPath()); + if (!published.equals(path.rawPath()) + && !isStoredProviderMetadataRevision(runtime, path, state)) { + throw notFound(path.rawPath()); + } return assets.serve(runtime, path.rawPath(), headOnly); } + private boolean isStoredProviderMetadataRevision( + RepositoryRuntime runtime, TerraformPath path, TerraformRegistryDao.ProviderState state) { + String prefix = "v1/providers/" + path.namespace() + "/" + path.name() + "/" + + path.version() + "/metadata-r"; + int separator = path.rawPath().indexOf('/', prefix.length()); + if (!path.rawPath().startsWith(prefix) || separator < 0) return false; + try { + long revision = Long.parseLong(path.rawPath().substring(prefix.length(), separator)); + return revision > 0 && revision <= state.revision() + && assets.find(runtime, path.rawPath()).isPresent(); + } catch (NumberFormatException e) { + return false; + } + } + private MavenResponse putModule( RepositoryRuntime runtime, TerraformPath path, InputStream body, String contentType, String actor, String ip) { + // Fail fast before buffering, then repeat this authoritative check while holding the shared + // coordinate lease so concurrent replicas cannot both publish different filenames. enforceModuleWrite(runtime, path); Path buffered = inspector.bufferAndInspect(body, path.filename(), true, null); - try (InputStream in = Files.newInputStream(buffered)) { - assets.store(runtime, path.rawPath(), in, contentType == null ? OCTET : contentType, - Map.of( - "terraformKind", "module-archive", - "namespace", path.namespace(), "name", path.name(), "system", path.system(), - "version", path.version(), "sha256Validated", true), actor, ip); - return MavenResponse.created(); + String leaseKey = publishLeaseKey( + "module", runtime.id(), path.namespace(), path.name(), path.system(), path.version()); + try (TerraformPublishLeaseManager.Lease lease = leases.acquire( + leaseKey, java.time.Duration.ofMinutes(5), java.time.Duration.ofSeconds(30))) { + enforceModuleWrite(runtime, path); + try (InputStream in = Files.newInputStream(buffered)) { + assets.store(runtime, path.rawPath(), in, contentType == null ? OCTET : contentType, + Map.of( + "terraformKind", "module-archive", + "namespace", path.namespace(), "name", path.name(), "system", path.system(), + "version", path.version(), "sha256Validated", true), actor, ip); + return MavenResponse.created(); + } } catch (IOException e) { throw new IllegalStateException("Failed storing Terraform module archive", e); } finally { @@ -253,7 +279,8 @@ private MavenResponse putProvider( throw new MavenExceptions.BadRequestException( "Provider filename must match " + expectedPrefix + "*.zip"); } - String leaseKey = runtime.id() + ":" + path.namespace() + ":" + path.name() + ":" + path.version(); + String leaseKey = publishLeaseKey( + "provider", runtime.id(), path.namespace(), path.name(), path.version()); try (TerraformPublishLeaseManager.Lease lease = leases.acquire( leaseKey, java.time.Duration.ofMinutes(5), java.time.Duration.ofSeconds(30))) { List current = registry.listProviderPlatforms( @@ -820,6 +847,10 @@ private static String sourceBindingKey(String assetPath) { return "asset:sha256:" + sha256(assetPath); } + private static String publishLeaseKey(String kind, long repositoryId, String... coordinates) { + return kind + ":" + repositoryId + ":" + sha256(String.join("\0", coordinates)); + } + private static MavenExceptions.MavenNotFoundException notFound(String path) { return new MavenExceptions.MavenNotFoundException(path); } diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/upload/ComponentUploadErrorAdvice.java b/server/src/main/java/com/github/klboke/kkrepo/server/upload/ComponentUploadErrorAdvice.java index 039438fb..0a0b02f0 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/upload/ComponentUploadErrorAdvice.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/upload/ComponentUploadErrorAdvice.java @@ -37,7 +37,8 @@ public ResponseEntity> mavenNotFound(MavenExceptions.MavenNo @ExceptionHandler({MavenExceptions.LayoutPolicyViolation.class, MavenExceptions.VersionPolicyViolation.class, - MavenExceptions.WritePolicyDenied.class}) + MavenExceptions.WritePolicyDenied.class, + MavenExceptions.BadRequestException.class}) public ResponseEntity> mavenBadRequest(RuntimeException e) { return body(HttpStatus.BAD_REQUEST, e.getMessage()); } diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java index 18afb84e..ef707e46 100644 --- a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java +++ b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java @@ -107,6 +107,7 @@ void servesAndPublishesHostedModulesWithNexusUrlTokenSemantics() throws Exceptio Path buffered = Files.createTempFile("terraform-module-service-test", ".zip"); Files.writeString(buffered, "module"); when(inspector.bufferAndInspect(any(), eq("new.zip"), eq(true), eq(null))).thenReturn(buffered); + when(registry.tryAcquirePublishLease(anyString(), anyString(), any())).thenReturn(true); when(assets.find(hosted, "v1/modules/acme/network/aws/3.0.0/new.zip")) .thenReturn(Optional.empty(), Optional.of(asset(3, hosted, 13L, "v1/modules/acme/network/aws/3.0.0/new.zip"))); @@ -117,6 +118,8 @@ void servesAndPublishesHostedModulesWithNexusUrlTokenSemantics() throws Exceptio null, null, "alice", "127.0.0.1"); assertEquals(201, published.status()); assertFalse(Files.exists(buffered)); + verify(registry).tryAcquirePublishLease(anyString(), anyString(), any()); + verify(registry).releasePublishLease(anyString(), anyString()); verify(assets).store(eq(hosted), eq("v1/modules/acme/network/aws/3.0.0/new.zip"), any(), eq("application/octet-stream"), any(), eq("alice"), eq("127.0.0.1")); @@ -146,18 +149,43 @@ void servesAndPublishesHostedModulesWithNexusUrlTokenSemantics() throws Exceptio new ByteArrayInputStream(new byte[0]), null, null, "alice", null)); } + @Test + void rechecksModuleCoordinateWhileHoldingSharedPublishLease() throws Exception { + RepositoryRuntime hosted = runtime(5, "terraform", RepositoryType.HOSTED, null, List.of()); + TerraformPath upload = paths.parse("v1/modules/acme/network/aws/1.0.0/alternate.zip"); + AssetRecord concurrent = asset( + 6, hosted, 16L, "v1/modules/acme/network/aws/1.0.0/network.zip"); + Path buffered = Files.createTempFile("terraform-module-lease-test", ".zip"); + Files.writeString(buffered, "module"); + when(assets.list(hosted, "v1/modules/acme/network/aws/1.0.0/")) + .thenReturn(List.of(), List.of(concurrent)); + when(inspector.bufferAndInspect(any(), eq("alternate.zip"), eq(true), eq(null))) + .thenReturn(buffered); + when(registry.tryAcquirePublishLease(anyString(), anyString(), any())).thenReturn(true); + + assertThrows(MavenExceptions.WritePolicyDenied.class, + () -> service.put(hosted, upload, new ByteArrayInputStream(new byte[0]), + null, null, "alice", null)); + + assertFalse(Files.exists(buffered)); + verify(registry).releasePublishLease(anyString(), anyString()); + verify(assets, never()).store(eq(hosted), anyString(), any(), anyString(), any(), any(), any()); + } + @Test void servesHostedProviderMetadataArchivesAndSignatures() throws Exception { RepositoryRuntime hosted = runtime(10, "terraform", RepositoryType.HOSTED, null, List.of()); String archivePath = "v1/providers/acme/cloud/1.2.3/package/linux/" + "terraform-provider-cloud_1.2.3_linux_amd64.zip"; - String sumsPath = "v1/providers/acme/cloud/1.2.3/metadata-r1/" + String sumsPath = "v1/providers/acme/cloud/1.2.3/metadata-r2/" + "terraform-provider-cloud_1.2.3_SHA256SUMS"; String signaturePath = sumsPath + ".sig"; + String oldSumsPath = "v1/providers/acme/cloud/1.2.3/metadata-r1/" + + "terraform-provider-cloud_1.2.3_SHA256SUMS"; AssetRecord archive = asset(20, hosted, 21L, archivePath); TerraformRegistryDao.ProviderPlatform platform = platform(hosted, archivePath, "abc123", 1); TerraformRegistryDao.ProviderState state = new TerraformRegistryDao.ProviderState( - hosted.id(), "acme", "cloud", "1.2.3", 1, sumsPath, signaturePath, 2, Instant.now()); + hosted.id(), "acme", "cloud", "1.2.3", 2, sumsPath, signaturePath, 2, Instant.now()); when(assets.list(eq(hosted), anyString())).thenReturn(List.of(archive)); when(registry.findProviderState(hosted.id(), "acme", "cloud", "1.2.3")) .thenReturn(Optional.of(state)); @@ -166,6 +194,8 @@ void servesHostedProviderMetadataArchivesAndSignatures() throws Exception { when(registry.findSigningKey(hosted.id(), 2)).thenReturn(Optional.of( new TerraformRegistryDao.SigningKey(hosted.id(), 2, "0011223344556677", "encrypted", "PUBLIC KEY", Instant.now()))); + when(assets.find(hosted, oldSumsPath)).thenReturn(Optional.of( + asset(22, hosted, 23L, oldSumsPath))); Map versions = json(service.get( hosted, paths.parse("v1/providers/acme/cloud/versions"), BASE, false)); @@ -186,6 +216,10 @@ void servesHostedProviderMetadataArchivesAndSignatures() throws Exception { assertEquals(200, service.get(hosted, paths.parse(archivePath), BASE, false).status()); assertEquals(200, service.get(hosted, paths.parse(sumsPath), BASE, true).status()); assertEquals(200, service.get(hosted, paths.parse(signaturePath), BASE, false).status()); + assertEquals(200, service.get(hosted, paths.parse(oldSumsPath), BASE, false).status()); + assertThrows(MavenExceptions.MavenNotFoundException.class, + () -> service.get(hosted, paths.parse(oldSumsPath.replace("metadata-r1", "metadata-r3")), + BASE, false)); when(registry.listProviderPlatforms(hosted.id(), "acme", "cloud", "9.9.9")) .thenReturn(List.of()); diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/upload/ComponentUploadErrorAdviceTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/upload/ComponentUploadErrorAdviceTest.java index bef31dff..1676e93c 100644 --- a/server/src/test/java/com/github/klboke/kkrepo/server/upload/ComponentUploadErrorAdviceTest.java +++ b/server/src/test/java/com/github/klboke/kkrepo/server/upload/ComponentUploadErrorAdviceTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import com.github.klboke.kkrepo.server.cargo.CargoExceptions; +import com.github.klboke.kkrepo.server.maven.MavenExceptions; import java.util.Map; import org.junit.jupiter.api.Test; import org.springframework.http.HttpStatus; @@ -38,4 +39,13 @@ void cargoUploadMethodNotAllowedUsesMethodNotAllowed() { assertEquals(HttpStatus.METHOD_NOT_ALLOWED, response.getStatusCode()); assertEquals("Operation is only valid on hosted Cargo repositories", response.getBody().get("error")); } + + @Test + void terraformUploadBadRequestUsesUploadErrorBody() { + ResponseEntity> response = + advice.mavenBadRequest(new MavenExceptions.BadRequestException("Invalid Terraform archive")); + + assertEquals(HttpStatus.BAD_REQUEST, response.getStatusCode()); + assertEquals("Invalid Terraform archive", response.getBody().get("error")); + } } From 8f754e30fbb5a44f003b40cca729af3d02e67a39 Mon Sep 17 00:00:00 2001 From: kl Date: Wed, 15 Jul 2026 11:00:20 +0800 Subject: [PATCH 11/53] fix: isolate Terraform migration and discovery state --- ...erraformRepositoryDataMigrationWriter.java | 2 +- .../server/terraform/TerraformService.java | 61 ++++++++++++---- ...formRepositoryDataMigrationWriterTest.java | 19 ++--- .../terraform/TerraformServiceTest.java | 70 +++++++++++++++++-- 4 files changed, 124 insertions(+), 28 deletions(-) diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformRepositoryDataMigrationWriter.java b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformRepositoryDataMigrationWriter.java index 6ca54e77..09906573 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformRepositoryDataMigrationWriter.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformRepositoryDataMigrationWriter.java @@ -78,7 +78,7 @@ public MigratedAsset write( + ": expected " + source.size() + ", actual " + size); } try (InputStream replay = Files.newInputStream(buffered)) { - service.put( + service.putForMigration( runtime, target.path(), replay, diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java index d57790f7..24ccab9f 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java @@ -110,10 +110,39 @@ public MavenResponse put( String contentDisposition, String actor, String ip) { + return publish(runtime, path, body, contentType, contentDisposition, actor, ip, false); + } + + /** + * Replays a validated migration asset while preserving publication leases and coordinate + * uniqueness. A Nexus repository may be frozen with DENY, so migration alone treats DENY as + * ALLOW_ONCE instead of requiring an operator to mutate the imported repository configuration. + */ + MavenResponse putForMigration( + RepositoryRuntime runtime, + TerraformPath path, + InputStream body, + String contentType, + String contentDisposition, + String actor, + String ip) { + return publish(runtime, path, body, contentType, contentDisposition, actor, ip, true); + } + + private MavenResponse publish( + RepositoryRuntime runtime, + TerraformPath path, + InputStream body, + String contentType, + String contentDisposition, + String actor, + String ip, + boolean migration) { if (!runtime.isHosted()) throw new MavenExceptions.MethodNotAllowed("Terraform group/proxy repositories are read-only"); return switch (path.kind()) { - case MODULE_ARCHIVE -> putModule(runtime, path, body, contentType, actor, ip); - case PROVIDER_DOWNLOAD -> putProvider(runtime, path, body, contentType, contentDisposition, actor, ip); + case MODULE_ARCHIVE -> putModule(runtime, path, body, contentType, actor, ip, migration); + case PROVIDER_DOWNLOAD -> + putProvider(runtime, path, body, contentType, contentDisposition, actor, ip, migration); default -> throw new MavenExceptions.MethodNotAllowed("Unsupported Terraform PUT path: " + path.rawPath()); }; } @@ -243,16 +272,16 @@ private boolean isStoredProviderMetadataRevision( private MavenResponse putModule( RepositoryRuntime runtime, TerraformPath path, InputStream body, String contentType, - String actor, String ip) { + String actor, String ip, boolean migration) { // Fail fast before buffering, then repeat this authoritative check while holding the shared // coordinate lease so concurrent replicas cannot both publish different filenames. - enforceModuleWrite(runtime, path); + enforceModuleWrite(runtime, path, migration); Path buffered = inspector.bufferAndInspect(body, path.filename(), true, null); String leaseKey = publishLeaseKey( "module", runtime.id(), path.namespace(), path.name(), path.system(), path.version()); try (TerraformPublishLeaseManager.Lease lease = leases.acquire( leaseKey, java.time.Duration.ofMinutes(5), java.time.Duration.ofSeconds(30))) { - enforceModuleWrite(runtime, path); + enforceModuleWrite(runtime, path, migration); try (InputStream in = Files.newInputStream(buffered)) { assets.store(runtime, path.rawPath(), in, contentType == null ? OCTET : contentType, Map.of( @@ -270,8 +299,8 @@ private MavenResponse putModule( private MavenResponse putProvider( RepositoryRuntime runtime, TerraformPath path, InputStream body, String contentType, - String contentDisposition, String actor, String ip) { - enforceWrite(runtime, path.rawPath()); + String contentDisposition, String actor, String ip, boolean migration) { + enforceWrite(runtime, path.rawPath(), migration); String filename = filename(contentDisposition); String expectedPrefix = "terraform-provider-" + path.name() + "_" + path.version() + "_" + path.os() + "_" + path.arch(); @@ -605,7 +634,7 @@ private String remoteUrl(RepositoryRuntime runtime, TerraformPath path) { private String discovery(RepositoryRuntime runtime, String key) { String root = ensureSlash(runtime.proxyRemoteUrl()); String remote = root + ".well-known/terraform.json"; - String local = ".terraform/upstream/discovery.json"; + String local = ".terraform/upstream/discovery-" + sha256(remote) + ".json"; Map document = readJson(responseBytes( proxy.getMetadataFromUrl(runtime, local, remote, false))); String value = string(document.get(key)); @@ -627,17 +656,17 @@ private static String routePath(String localPath) { return ".terraform/routes/" + sha256(localPath) + ".json"; } - private void enforceWrite(RepositoryRuntime runtime, String path) { - String policy = runtime.writePolicy() == null ? "ALLOW_ONCE" : runtime.writePolicy().toUpperCase(Locale.ROOT); + private void enforceWrite(RepositoryRuntime runtime, String path, boolean migration) { + String policy = effectiveWritePolicy(runtime, migration); if ("DENY".equals(policy)) throw new MavenExceptions.WritePolicyDenied("Repository write policy is DENY"); if ("ALLOW_ONCE".equals(policy) && assets.find(runtime, path).isPresent()) { throw new MavenExceptions.WritePolicyDenied("Terraform coordinate already exists"); } } - private void enforceModuleWrite(RepositoryRuntime runtime, TerraformPath path) { - String policy = runtime.writePolicy() == null - ? "ALLOW_ONCE" : runtime.writePolicy().toUpperCase(Locale.ROOT); + private void enforceModuleWrite( + RepositoryRuntime runtime, TerraformPath path, boolean migration) { + String policy = effectiveWritePolicy(runtime, migration); if ("DENY".equals(policy)) { throw new MavenExceptions.WritePolicyDenied("Repository write policy is DENY"); } @@ -655,6 +684,12 @@ private void enforceModuleWrite(RepositoryRuntime runtime, TerraformPath path) { throw new MavenExceptions.WritePolicyDenied("Terraform module version already exists"); } + private static String effectiveWritePolicy(RepositoryRuntime runtime, boolean migration) { + String policy = runtime.writePolicy() == null + ? "ALLOW_ONCE" : runtime.writePolicy().toUpperCase(Locale.ROOT); + return migration && "DENY".equals(policy) ? "ALLOW_ONCE" : policy; + } + private String filename(String contentDisposition) { return contentDispositionFilename(contentDisposition); } diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformRepositoryDataMigrationWriterTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformRepositoryDataMigrationWriterTest.java index 3b9e1978..b89f6bc7 100644 --- a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformRepositoryDataMigrationWriterTest.java +++ b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformRepositoryDataMigrationWriterTest.java @@ -66,7 +66,8 @@ void reusesAlreadyMigratedModuleAndValidatesItsBlob() { assertEquals(101L, migrated.assetId()); assertEquals(102L, migrated.assetBlobId()); assertEquals("module-object", migrated.assetBlobObjectKey()); - verify(fixture.service(), never()).put(any(), any(), any(), any(), any(), any(), any()); + verify(fixture.service(), never()) + .putForMigration(any(), any(), any(), any(), any(), any(), any()); RepositoryDataMigrationAssetRecord wrongSize = source(path, 8L, null, null, null); assertThrows(IllegalStateException.class, @@ -101,7 +102,8 @@ void reusesPublishedProviderPlatformOnlyAfterSharedStateConfirmsPublication() { assertEquals(201L, migrated.assetId()); assertEquals("provider-object", migrated.assetBlobObjectKey()); - verify(fixture.service(), never()).put(any(), any(), any(), any(), any(), any(), any()); + verify(fixture.service(), never()) + .putForMigration(any(), any(), any(), any(), any(), any(), any()); } @Test @@ -115,7 +117,7 @@ void buffersAndReplaysModuleThroughNormalPublicationPath() { when(fixture.assets().find(fixture.runtime(), path)) .thenReturn(Optional.empty(), Optional.of(stored)); when(fixture.assets().blob(stored)).thenReturn(blob(302, content.length, "module-new")); - when(fixture.service().put(any(), any(), any(), any(), any(), any(), any())) + when(fixture.service().putForMigration(any(), any(), any(), any(), any(), any(), any())) .thenReturn(MavenResponse.created()); TerraformRepositoryDataMigrationWriter.MigratedAsset migrated = fixture.writer().write( @@ -123,7 +125,7 @@ void buffersAndReplaysModuleThroughNormalPublicationPath() { assertEquals(301L, migrated.assetId()); ArgumentCaptor pathCaptor = ArgumentCaptor.forClass(TerraformPath.class); - verify(fixture.service()).put( + verify(fixture.service()).putForMigration( eq(fixture.runtime()), pathCaptor.capture(), any(InputStream.class), eq("application/x-terraform-module"), eq(null), eq("nexus-user"), eq("10.0.0.2")); assertEquals(TerraformPath.Kind.MODULE_ARCHIVE, pathCaptor.getValue().kind()); @@ -144,14 +146,14 @@ void mapsNexusProviderArchiveToCanonicalPackagePathAndDefaultsMetadata() { when(fixture.registry().listProviderPlatforms( fixture.runtime().id(), "kkrepo", "fixture", "1.2.3")).thenReturn(List.of()); when(fixture.assets().blob(stored)).thenReturn(blob(402, content.length, "provider-new")); - when(fixture.service().put(any(), any(), any(), any(), any(), any(), any())) + when(fixture.service().putForMigration(any(), any(), any(), any(), any(), any(), any())) .thenReturn(MavenResponse.created()); fixture.writer().write( fixture.repository(), source, new ByteArrayInputStream(content), " ", true); ArgumentCaptor pathCaptor = ArgumentCaptor.forClass(TerraformPath.class); - verify(fixture.service()).put( + verify(fixture.service()).putForMigration( eq(fixture.runtime()), pathCaptor.capture(), any(InputStream.class), eq("application/octet-stream"), eq("attachment; filename=\"" + filename + "\""), eq("nexus-migration"), eq(null)); @@ -188,9 +190,10 @@ void rejectsInvalidTargetsSizeMismatchAndMissingPublishedAsset() { assertThrows(IllegalStateException.class, () -> fixture.writer().write(fixture.repository(), source(modulePath, 99L, null, null, null), new ByteArrayInputStream(content), null, true)); - verify(fixture.service(), never()).put(any(), any(), any(), any(), any(), any(), any()); + verify(fixture.service(), never()) + .putForMigration(any(), any(), any(), any(), any(), any(), any()); - when(fixture.service().put(any(), any(), any(), any(), any(), any(), any())) + when(fixture.service().putForMigration(any(), any(), any(), any(), any(), any(), any())) .thenReturn(MavenResponse.created()); assertThrows(IllegalStateException.class, () -> fixture.writer().write(fixture.repository(), source(modulePath, 6L, null, null, null), diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java index ef707e46..6e0f3f2d 100644 --- a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java +++ b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java @@ -172,6 +172,31 @@ void rechecksModuleCoordinateWhileHoldingSharedPublishLease() throws Exception { verify(assets, never()).store(eq(hosted), anyString(), any(), anyString(), any(), any(), any()); } + @Test + void migrationPublishesModuleThroughDenyPolicyWithSharedLease() throws Exception { + RepositoryRuntime denied = runtime( + 6, "terraform-denied", RepositoryType.HOSTED, null, List.of(), "DENY"); + TerraformPath upload = paths.parse("v1/modules/acme/network/aws/1.0.0/module.zip"); + Path buffered = Files.createTempFile("terraform-module-migration-test", ".zip"); + Files.writeString(buffered, "module"); + when(assets.list(denied, "v1/modules/acme/network/aws/1.0.0/")) + .thenReturn(List.of()); + when(inspector.bufferAndInspect(any(), eq("module.zip"), eq(true), eq(null))) + .thenReturn(buffered); + when(registry.tryAcquirePublishLease(anyString(), anyString(), any())).thenReturn(true); + + MavenResponse response = service.putForMigration( + denied, upload, new ByteArrayInputStream(new byte[0]), + "application/zip", null, "nexus-migration", null); + + assertEquals(201, response.status()); + assertFalse(Files.exists(buffered)); + verify(assets).store( + eq(denied), eq(upload.rawPath()), any(), eq("application/zip"), any(), + eq("nexus-migration"), eq(null)); + verify(registry).releasePublishLease(anyString(), anyString()); + } + @Test void servesHostedProviderMetadataArchivesAndSignatures() throws Exception { RepositoryRuntime hosted = runtime(10, "terraform", RepositoryType.HOSTED, null, List.of()); @@ -229,8 +254,9 @@ void servesHostedProviderMetadataArchivesAndSignatures() throws Exception { } @Test - void publishesProviderPlatformAtomicallyUnderSharedLease() throws Exception { - RepositoryRuntime hosted = runtime(30, "terraform", RepositoryType.HOSTED, null, List.of()); + void publishesMigratedProviderPlatformAtomicallyThroughDenyPolicy() throws Exception { + RepositoryRuntime hosted = runtime( + 30, "terraform", RepositoryType.HOSTED, null, List.of(), "DENY"); TerraformPath upload = paths.parse("v1/providers/acme/cloud/1.2.3/download/linux/amd64"); String filename = "terraform-provider-cloud_1.2.3_linux_amd64.zip"; String archivePath = "v1/providers/acme/cloud/1.2.3/package/linux/" + filename; @@ -250,7 +276,7 @@ void publishesProviderPlatformAtomicallyUnderSharedLease() throws Exception { new TerraformSigningService.SigningMaterial(4, "AABBCCDDEEFF0011", "public", "private", "pass")); when(signing.sign(any(), any())).thenReturn("signature".getBytes(StandardCharsets.UTF_8)); - MavenResponse response = service.put( + MavenResponse response = service.putForMigration( hosted, upload, new ByteArrayInputStream("provider".getBytes(StandardCharsets.UTF_8)), "application/zip", "attachment; filename=\"" + filename + "\"", "alice", "127.0.0.1"); assertEquals(201, response.status()); @@ -264,7 +290,7 @@ hosted, upload, new ByteArrayInputStream("provider".getBytes(StandardCharsets.UT verify(assets, atLeastOnce()).storeBytes(eq(hosted), anyString(), any(), anyString(), any()); assertThrows(MavenExceptions.BadRequestException.class, - () -> service.put(hosted, upload, new ByteArrayInputStream(new byte[0]), null, + () -> service.putForMigration(hosted, upload, new ByteArrayInputStream(new byte[0]), null, "attachment; filename=\"wrong.zip\"", "alice", null)); } @@ -305,7 +331,7 @@ void rewritesAndVerifiesProxyProviderMetadataAndRoutes() throws Exception { .thenAnswer(invocation -> { String local = invocation.getArgument(1); String remote = invocation.getArgument(2); - if (local.equals(".terraform/upstream/discovery.json")) { + if (local.startsWith(".terraform/upstream/discovery-")) { return response(mapper.writeValueAsBytes(Map.of( "modules.v1", "/api/modules/", "providers.v1", "/api/providers/")), "application/json"); } @@ -482,7 +508,7 @@ void rejectsMalformedProxyProviderMetadata() throws Exception { when(proxy.getMetadataFromUrl(eq(proxyRuntime), anyString(), anyString(), anyBoolean())) .thenAnswer(invocation -> { String local = invocation.getArgument(1); - if (local.equals(".terraform/upstream/discovery.json")) { + if (local.startsWith(".terraform/upstream/discovery-")) { return response(mapper.writeValueAsBytes(Map.of("providers.v1", "/v1/providers/")), "application/json"); } @@ -495,6 +521,38 @@ void rejectsMalformedProxyProviderMetadata() throws Exception { verify(signatureVerifier, never()).verify(any(), any(), any()); } + @Test + void keysDiscoveryMetadataByConfiguredRemoteUrl() throws Exception { + RepositoryRuntime original = runtime( + 71, "terraform-proxy", RepositoryType.PROXY, + "https://registry-one.example", List.of()); + RepositoryRuntime changed = runtime( + 71, "terraform-proxy", RepositoryType.PROXY, + "https://registry-two.example", List.of()); + Map discoveryCachePaths = new java.util.LinkedHashMap<>(); + when(proxy.getMetadataFromUrl(any(), anyString(), anyString(), anyBoolean())) + .thenAnswer(invocation -> { + String local = invocation.getArgument(1); + String remote = invocation.getArgument(2); + if (remote.endsWith("/.well-known/terraform.json")) { + discoveryCachePaths.put(remote, local); + return response(mapper.writeValueAsBytes(Map.of("modules.v1", "/v1/modules/")), + "application/json"); + } + return response(mapper.writeValueAsBytes(Map.of("modules", List.of())), + "application/json"); + }); + + service.get(original, paths.parse("v1/modules/acme/network/aws/versions"), BASE, false); + service.get(changed, paths.parse("v1/modules/acme/network/aws/versions"), BASE, false); + + assertEquals(2, discoveryCachePaths.size()); + assertEquals(2, Set.copyOf(discoveryCachePaths.values()).size()); + assertTrue(discoveryCachePaths.values().stream() + .allMatch(path -> path.startsWith(".terraform/upstream/discovery-") + && path.endsWith(".json"))); + } + private Map json(MavenResponse response) throws Exception { try (var body = response.body()) { return mapper.readValue(body, new TypeReference<>() {}); From 7ee14020af44dc16afb90872bcd33c85dd0c899f Mon Sep 17 00:00:00 2001 From: kl Date: Wed, 15 Jul 2026 11:17:18 +0800 Subject: [PATCH 12/53] fix: filter Terraform component search --- .../kkrepo/server/browse/ComponentSearchController.java | 1 + .../browse/ComponentSearchControllerSecurityTest.java | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/browse/ComponentSearchController.java b/server/src/main/java/com/github/klboke/kkrepo/server/browse/ComponentSearchController.java index dbd5ace5..eb6a3863 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/browse/ComponentSearchController.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/browse/ComponentSearchController.java @@ -96,6 +96,7 @@ private static RepositoryFormat parseFormat(String value) { case "go" -> RepositoryFormat.GO; case "pub" -> RepositoryFormat.PUB; case "composer" -> RepositoryFormat.COMPOSER; + case "terraform" -> RepositoryFormat.TERRAFORM; case "raw" -> RepositoryFormat.RAW; default -> null; }; diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/browse/ComponentSearchControllerSecurityTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/browse/ComponentSearchControllerSecurityTest.java index 357f3737..58f0f57f 100644 --- a/server/src/test/java/com/github/klboke/kkrepo/server/browse/ComponentSearchControllerSecurityTest.java +++ b/server/src/test/java/com/github/klboke/kkrepo/server/browse/ComponentSearchControllerSecurityTest.java @@ -132,7 +132,7 @@ void searchCanUseAnonymousSubjectWhenAnonymousAccessIsConfigured() { } @Test - void searchParsesNugetPubRubygemsAndYumFormats() { + void searchParsesNugetPubRubygemsYumAndTerraformFormats() { StubComponentDao components = new StubComponentDao(); RecordingSecurityService security = new RecordingSecurityService(permission -> AccessDecision.allow()); ComponentSearchController controller = controller(components, subject("alice"), null, security); @@ -141,8 +141,11 @@ void searchParsesNugetPubRubygemsAndYumFormats() { controller.search(null, "pub", null, request("GET", "/internal/search/components")); controller.search(null, "rubygems", null, request("GET", "/internal/search/components")); controller.search(null, "yum", null, request("GET", "/internal/search/components")); + controller.search(null, "terraform", null, request("GET", "/internal/search/components")); - assertEquals(List.of("|nuget|300", "|pub|300", "|rubygems|300", "|yum|300"), components.calls); + assertEquals( + List.of("|nuget|300", "|pub|300", "|rubygems|300", "|yum|300", "|terraform|300"), + components.calls); } @Test From 140008845a12136056365a89100d0607e04eb480 Mon Sep 17 00:00:00 2001 From: kl Date: Wed, 15 Jul 2026 11:39:30 +0800 Subject: [PATCH 13/53] fix: recover Terraform provider publication state --- .../server/terraform/TerraformService.java | 47 +++++++++++-------- .../terraform/TerraformServiceTest.java | 23 +++++++-- 2 files changed, 48 insertions(+), 22 deletions(-) diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java index 24ccab9f..e586a2d2 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java @@ -198,22 +198,30 @@ private Map providerVersions(RepositoryRuntime runtime, Terrafor } List> values = new ArrayList<>(); for (String version : TerraformVersions.descending(versions)) { - List> platforms = registry.listProviderPlatforms( - runtime.id(), request.namespace(), request.name(), version).stream() + List> platforms = publishedProviderPlatforms( + runtime, request.namespace(), request.name(), version).stream() .map(row -> Map.of("os", row.os(), "arch", row.arch())) .toList(); + if (platforms.isEmpty()) continue; values.add(Map.of("version", version, "protocols", List.of(PROTOCOLS), "platforms", platforms)); } return Map.of("versions", values); } + private List publishedProviderPlatforms( + RepositoryRuntime runtime, String namespace, String name, String version) { + return registry.listProviderPlatforms(runtime.id(), namespace, name, version).stream() + .filter(platform -> assets.find(runtime, platform.assetPath()).isPresent()) + .toList(); + } + private MavenResponse providerDownload( RepositoryRuntime runtime, TerraformPath request, RequestUrls urls, boolean headOnly) { TerraformRegistryDao.ProviderState state = registry.findProviderState( runtime.id(), request.namespace(), request.name(), request.version()) .orElseThrow(() -> notFound(request.rawPath())); - TerraformRegistryDao.ProviderPlatform platform = registry.listProviderPlatforms( - runtime.id(), request.namespace(), request.name(), request.version()).stream() + TerraformRegistryDao.ProviderPlatform platform = publishedProviderPlatforms( + runtime, request.namespace(), request.name(), request.version()).stream() .filter(row -> row.os().equals(request.os()) && row.arch().equals(request.arch())) .findFirst().orElseThrow(() -> notFound(request.rawPath())); TerraformRegistryDao.SigningKey key = registry.findSigningKey(runtime.id(), state.signingKeyRevision()) @@ -234,8 +242,8 @@ private MavenResponse providerDownload( private MavenResponse servePublishedProviderArchive( RepositoryRuntime runtime, TerraformPath path, boolean headOnly) { - boolean published = registry.listProviderPlatforms( - runtime.id(), path.namespace(), path.name(), path.version()).stream() + boolean published = publishedProviderPlatforms( + runtime, path.namespace(), path.name(), path.version()).stream() .anyMatch(row -> row.assetPath().equals(path.rawPath())); if (!published) throw notFound(path.rawPath()); return assets.serve(runtime, path.rawPath(), headOnly); @@ -312,25 +320,26 @@ private MavenResponse putProvider( "provider", runtime.id(), path.namespace(), path.name(), path.version()); try (TerraformPublishLeaseManager.Lease lease = leases.acquire( leaseKey, java.time.Duration.ofMinutes(5), java.time.Duration.ofSeconds(30))) { - List current = registry.listProviderPlatforms( - runtime.id(), path.namespace(), path.name(), path.version()); + List current = publishedProviderPlatforms( + runtime, path.namespace(), path.name(), path.version()); boolean exists = current.stream().anyMatch(row -> row.os().equals(path.os()) && row.arch().equals(path.arch())); if (exists) throw new MavenExceptions.WritePolicyDenied("Terraform provider platform already exists"); long revision = registry.findProviderState(runtime.id(), path.namespace(), path.name(), path.version()) .map(state -> state.revision() + 1).orElse(1L); String assetPath = "v1/providers/" + path.namespace() + "/" + path.name() + "/" + path.version() + "/package/" + path.os() + "/" + filename; - if (assets.find(runtime, assetPath).isEmpty()) { - Path buffered = inspector.bufferAndInspect(body, filename, false, path.name()); - try (InputStream in = Files.newInputStream(buffered)) { - assets.store(runtime, assetPath, in, contentType == null ? OCTET : contentType, - Map.of( - "terraformKind", "provider-archive", "namespace", path.namespace(), "type", path.name(), - "version", path.version(), "os", path.os(), "arch", path.arch(), - "protocols", List.of(PROTOCOLS), "publishState", "STAGING", "revision", revision), actor, ip); - } finally { - try { Files.deleteIfExists(buffered); } catch (IOException ignored) {} - } + // Always validate and persist the incoming archive. A prior publication attempt may have + // failed after storing an orphaned STAGING asset but before committing the platform row; + // reusing that blob would publish stale content instead of the operator's retry. + Path buffered = inspector.bufferAndInspect(body, filename, false, path.name()); + try (InputStream in = Files.newInputStream(buffered)) { + assets.store(runtime, assetPath, in, contentType == null ? OCTET : contentType, + Map.of( + "terraformKind", "provider-archive", "namespace", path.namespace(), "type", path.name(), + "version", path.version(), "os", path.os(), "arch", path.arch(), + "protocols", List.of(PROTOCOLS), "publishState", "STAGING", "revision", revision), actor, ip); + } finally { + try { Files.deleteIfExists(buffered); } catch (IOException ignored) {} } AssetRecord stored = assets.find(runtime, assetPath).orElseThrow(() -> notFound(assetPath)); AssetBlobRecord blob = assets.blob(stored); diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java index 6e0f3f2d..dbc45d0e 100644 --- a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java +++ b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java @@ -207,25 +207,31 @@ void servesHostedProviderMetadataArchivesAndSignatures() throws Exception { String signaturePath = sumsPath + ".sig"; String oldSumsPath = "v1/providers/acme/cloud/1.2.3/metadata-r1/" + "terraform-provider-cloud_1.2.3_SHA256SUMS"; + String deletedPath = "v1/providers/acme/cloud/1.2.3/package/darwin/" + + "terraform-provider-cloud_1.2.3_darwin_arm64.zip"; AssetRecord archive = asset(20, hosted, 21L, archivePath); TerraformRegistryDao.ProviderPlatform platform = platform(hosted, archivePath, "abc123", 1); + TerraformRegistryDao.ProviderPlatform deleted = + providerPlatform(hosted, deletedPath, "darwin", "arm64"); TerraformRegistryDao.ProviderState state = new TerraformRegistryDao.ProviderState( hosted.id(), "acme", "cloud", "1.2.3", 2, sumsPath, signaturePath, 2, Instant.now()); when(assets.list(eq(hosted), anyString())).thenReturn(List.of(archive)); when(registry.findProviderState(hosted.id(), "acme", "cloud", "1.2.3")) .thenReturn(Optional.of(state)); when(registry.listProviderPlatforms(hosted.id(), "acme", "cloud", "1.2.3")) - .thenReturn(List.of(platform)); + .thenReturn(List.of(platform, deleted)); when(registry.findSigningKey(hosted.id(), 2)).thenReturn(Optional.of( new TerraformRegistryDao.SigningKey(hosted.id(), 2, "0011223344556677", "encrypted", "PUBLIC KEY", Instant.now()))); when(assets.find(hosted, oldSumsPath)).thenReturn(Optional.of( asset(22, hosted, 23L, oldSumsPath))); + when(assets.find(hosted, archivePath)).thenReturn(Optional.of(archive)); Map versions = json(service.get( hosted, paths.parse("v1/providers/acme/cloud/versions"), BASE, false)); assertTrue(versions.toString().contains("linux")); assertTrue(versions.toString().contains("amd64")); + assertFalse(versions.toString().contains("darwin")); Map download = json(service.get( hosted, paths.parse("v1/providers/acme/cloud/1.2.3/download/linux/amd64"), BASE, @@ -236,6 +242,9 @@ void servesHostedProviderMetadataArchivesAndSignatures() throws Exception { assertFalse(service.get( hosted, paths.parse("v1/providers/acme/cloud/1.2.3/download/linux/amd64"), BASE, null, true).hasBody()); + assertThrows(MavenExceptions.MavenNotFoundException.class, + () -> service.get(hosted, + paths.parse("v1/providers/acme/cloud/1.2.3/download/darwin/arm64"), BASE, false)); when(assets.serve(eq(hosted), anyString(), anyBoolean())).thenReturn(MavenResponse.noBody(200)); assertEquals(200, service.get(hosted, paths.parse(archivePath), BASE, false).status()); @@ -254,7 +263,7 @@ void servesHostedProviderMetadataArchivesAndSignatures() throws Exception { } @Test - void publishesMigratedProviderPlatformAtomicallyThroughDenyPolicy() throws Exception { + void reinspectsOrphanedMigratedProviderBeforePublishingThroughDenyPolicy() throws Exception { RepositoryRuntime hosted = runtime( 30, "terraform", RepositoryType.HOSTED, null, List.of(), "DENY"); TerraformPath upload = paths.parse("v1/providers/acme/cloud/1.2.3/download/linux/amd64"); @@ -270,7 +279,7 @@ void publishesMigratedProviderPlatformAtomicallyThroughDenyPolicy() throws Excep when(registry.findProviderState(hosted.id(), "acme", "cloud", "1.2.3")) .thenReturn(Optional.empty()); when(inspector.bufferAndInspect(any(), eq(filename), eq(false), eq("cloud"))).thenReturn(buffered); - when(assets.find(hosted, archivePath)).thenReturn(Optional.empty(), Optional.of(archive)); + when(assets.find(hosted, archivePath)).thenReturn(Optional.of(archive)); when(assets.blob(archive)).thenReturn(blob); when(signing.active(hosted)).thenReturn( new TerraformSigningService.SigningMaterial(4, "AABBCCDDEEFF0011", "public", "private", "pass")); @@ -287,6 +296,10 @@ hosted, upload, new ByteArrayInputStream("provider".getBytes(StandardCharsets.UT verify(registry).publishProvider(any(TerraformRegistryDao.ProviderPlatform.class), state.capture()); assertEquals(1, state.getValue().revision()); assertEquals(4, state.getValue().signingKeyRevision()); + verify(inspector).bufferAndInspect(any(), eq(filename), eq(false), eq("cloud")); + verify(assets).store( + eq(hosted), eq(archivePath), any(), eq("application/zip"), any(), + eq("alice"), eq("127.0.0.1")); verify(assets, atLeastOnce()).storeBytes(eq(hosted), anyString(), any(), anyString(), any()); assertThrows(MavenExceptions.BadRequestException.class, @@ -488,6 +501,10 @@ void mergesPlatformsForTheSameProviderVersionAcrossGroupMembers() throws Excepti .thenReturn(List.of(providerPlatform(first, linuxPath, "linux", "amd64"))); when(registry.listProviderPlatforms(second.id(), "acme", "cloud", "1.2.3")) .thenReturn(List.of(providerPlatform(second, darwinPath, "darwin", "arm64"))); + when(assets.find(first, linuxPath)).thenReturn(Optional.of( + asset(53, first, 63L, linuxPath))); + when(assets.find(second, darwinPath)).thenReturn(Optional.of( + asset(54, second, 64L, darwinPath))); Map merged = json(service.get( group, paths.parse("v1/providers/acme/cloud/versions"), BASE, false)); From 5b07cda997531bdd62afbf4ddfe6a23ecdc192f7 Mon Sep 17 00:00:00 2001 From: kl Date: Wed, 15 Jul 2026 11:58:00 +0800 Subject: [PATCH 14/53] fix: accept Terraform JSON module syntax --- .../terraform/TerraformArchiveInspector.java | 4 ++-- .../terraform/TerraformArchiveInspectorTest.java | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformArchiveInspector.java b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformArchiveInspector.java index 5859632f..68fca7cf 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformArchiveInspector.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformArchiveInspector.java @@ -47,7 +47,7 @@ Path bufferAndInspect(InputStream body, String filename, boolean module, String : inspectTar(file, filename, module, providerName); if (!valid) { throw bad(module - ? "Terraform module archive must contain at least one .tf file" + ? "Terraform module archive must contain at least one .tf or .tf.json file" : "Terraform provider archive does not contain the expected provider binary"); } return file; @@ -128,7 +128,7 @@ private static void validateEntry(String rawName, boolean link, boolean director private static boolean matches(String name, boolean module, String providerName) { String leaf = name.replace('\\', '/'); leaf = leaf.substring(leaf.lastIndexOf('/') + 1); - return module ? leaf.endsWith(".tf") + return module ? leaf.endsWith(".tf") || leaf.endsWith(".tf.json") : leaf.startsWith("terraform-provider-" + providerName); } diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformArchiveInspectorTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformArchiveInspectorTest.java index b13c8220..5fd97431 100644 --- a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformArchiveInspectorTest.java +++ b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformArchiveInspectorTest.java @@ -76,6 +76,21 @@ void acceptsZipModulesAndProviders() throws Exception { } } + @Test + void acceptsJsonSyntaxOnlyModuleArchive() throws Exception { + Path module = null; + try { + module = inspector.bufferAndInspect( + new ByteArrayInputStream(zipEntry( + "fixture/main.tf.json", "{\"terraform\":{}}".getBytes(StandardCharsets.UTF_8))), + "module.zip", true, null); + + assertTrue(Files.size(module) > 0); + } finally { + if (module != null) Files.deleteIfExists(module); + } + } + @Test void rejectsArchivesWithoutExpectedContentAndUnsafeZipEntries() throws Exception { byte[] noModule = zipEntry("fixture/README.md", "readme".getBytes(StandardCharsets.UTF_8)); From a7fe034e37abfc56d232d6deeb6d45ad161c0809 Mon Sep 17 00:00:00 2001 From: kl Date: Wed, 15 Jul 2026 12:15:42 +0800 Subject: [PATCH 15/53] fix: inspect provider uploads before leasing --- .../klboke/kkrepo/server/terraform/TerraformService.java | 8 +++++--- .../kkrepo/server/terraform/TerraformServiceTest.java | 5 ++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java index e586a2d2..6c17cfed 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java @@ -316,6 +316,9 @@ private MavenResponse putProvider( throw new MavenExceptions.BadRequestException( "Provider filename must match " + expectedPrefix + "*.zip"); } + // Buffer and inspect before acquiring the fixed-TTL lease. Slow uploads must not consume the + // lease lifetime and allow another replica to publish the same provider version concurrently. + Path buffered = inspector.bufferAndInspect(body, filename, false, path.name()); String leaseKey = publishLeaseKey( "provider", runtime.id(), path.namespace(), path.name(), path.version()); try (TerraformPublishLeaseManager.Lease lease = leases.acquire( @@ -331,15 +334,12 @@ private MavenResponse putProvider( // Always validate and persist the incoming archive. A prior publication attempt may have // failed after storing an orphaned STAGING asset but before committing the platform row; // reusing that blob would publish stale content instead of the operator's retry. - Path buffered = inspector.bufferAndInspect(body, filename, false, path.name()); try (InputStream in = Files.newInputStream(buffered)) { assets.store(runtime, assetPath, in, contentType == null ? OCTET : contentType, Map.of( "terraformKind", "provider-archive", "namespace", path.namespace(), "type", path.name(), "version", path.version(), "os", path.os(), "arch", path.arch(), "protocols", List.of(PROTOCOLS), "publishState", "STAGING", "revision", revision), actor, ip); - } finally { - try { Files.deleteIfExists(buffered); } catch (IOException ignored) {} } AssetRecord stored = assets.find(runtime, assetPath).orElseThrow(() -> notFound(assetPath)); AssetBlobRecord blob = assets.blob(stored); @@ -369,6 +369,8 @@ private MavenResponse putProvider( return MavenResponse.created(); } catch (IOException e) { throw new IllegalStateException("Failed storing Terraform provider archive", e); + } finally { + try { Files.deleteIfExists(buffered); } catch (IOException ignored) {} } } diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java index dbc45d0e..9746ef71 100644 --- a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java +++ b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java @@ -10,6 +10,7 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -290,13 +291,15 @@ hosted, upload, new ByteArrayInputStream("provider".getBytes(StandardCharsets.UT "application/zip", "attachment; filename=\"" + filename + "\"", "alice", "127.0.0.1"); assertEquals(201, response.status()); assertFalse(Files.exists(buffered)); + var publicationOrder = inOrder(inspector, registry); + publicationOrder.verify(inspector).bufferAndInspect(any(), eq(filename), eq(false), eq("cloud")); + publicationOrder.verify(registry).tryAcquirePublishLease(anyString(), anyString(), any()); verify(registry).releasePublishLease(anyString(), anyString()); ArgumentCaptor state = ArgumentCaptor.forClass(TerraformRegistryDao.ProviderState.class); verify(registry).publishProvider(any(TerraformRegistryDao.ProviderPlatform.class), state.capture()); assertEquals(1, state.getValue().revision()); assertEquals(4, state.getValue().signingKeyRevision()); - verify(inspector).bufferAndInspect(any(), eq(filename), eq(false), eq("cloud")); verify(assets).store( eq(hosted), eq(archivePath), any(), eq("application/zip"), any(), eq("alice"), eq("127.0.0.1")); From f2e556ec855daf7b7f2230c3fd4c71a671189dab Mon Sep 17 00:00:00 2001 From: kl Date: Wed, 15 Jul 2026 12:39:53 +0800 Subject: [PATCH 16/53] fix: discard invalid Terraform proxy uploads --- .../terraform/TerraformAssetSupport.java | 4 ++ .../server/terraform/TerraformService.java | 8 ++++ .../server/upload/ComponentUploadService.java | 13 ++++- .../terraform/TerraformAssetSupportTest.java | 2 + .../terraform/TerraformServiceTest.java | 8 +++- .../upload/ComponentUploadServiceTest.java | 47 +++++++++++++++++++ 6 files changed, 79 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformAssetSupport.java b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformAssetSupport.java index 95c2f0bf..3c32dfe1 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformAssetSupport.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformAssetSupport.java @@ -38,6 +38,10 @@ void storeBytes(RepositoryRuntime runtime, String path, byte[] body, String cont store(runtime, path, new ByteArrayInputStream(body), contentType, attributes, "terraform", null); } + void delete(RepositoryRuntime runtime, String path) { + hosted.deleteInternal(runtime, path); + } + Optional find(RepositoryRuntime runtime, String path) { return assets.findAssetByPath(runtime.id(), path); } diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java index 6c17cfed..62825511 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java @@ -501,6 +501,14 @@ private MavenResponse proxyRoute(RepositoryRuntime runtime, String localPath, bo AssetRecord asset = assets.find(runtime, localPath).orElseThrow(() -> notFound(localPath)); AssetBlobRecord blob = assets.blob(asset); if (blob == null || !expected.equalsIgnoreCase(blob.sha256())) { + if (response.hasBody()) { + try { + response.body().close(); + } catch (IOException ignored) { + // The checksum failure is the actionable upstream error. + } + } + assets.delete(runtime, localPath); throw new MavenExceptions.BadUpstreamException("Terraform provider checksum mismatch for " + localPath); } } diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/upload/ComponentUploadService.java b/server/src/main/java/com/github/klboke/kkrepo/server/upload/ComponentUploadService.java index 6fcf1df8..6ea797c6 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/upload/ComponentUploadService.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/upload/ComponentUploadService.java @@ -178,6 +178,13 @@ public UploadResult upload( if (!runtime.isHosted()) { throw new MavenExceptions.MethodNotAllowed("Upload is only valid on hosted repositories"); } + if (runtime.format() == RepositoryFormat.TERRAFORM + && rawFiles.values().stream() + .flatMap(List::stream) + .filter(file -> file != null && !file.isEmpty()) + .count() != 1) { + throw new UploadValidationException("Terraform upload requires exactly one archive"); + } String format = formatLabel(runtime.format()); NormalizedUpload upload = normalize(format, rawFields, rawFiles); List paths = switch (runtime.format()) { @@ -202,12 +209,14 @@ public UploadResult upload( private List uploadTerraform( RepositoryRuntime runtime, NormalizedUpload upload, String createdBy, String createdByIp) throws IOException { if (terraformService == null) throw new UploadValidationException("Terraform upload service is unavailable"); + if (upload.assets().size() != 1) { + throw new UploadValidationException("Terraform upload requires exactly one archive"); + } String kind = requireField(upload.fields(), "kind").trim().toLowerCase(Locale.ROOT); String namespace = requireField(upload.fields(), "namespace"); String name = requireField(upload.fields(), "name"); String version = requireField(upload.fields(), "version"); - AssetUpload asset = upload.assets().stream().findFirst() - .orElseThrow(() -> new UploadValidationException("Terraform upload requires one archive")); + AssetUpload asset = upload.assets().getFirst(); String filename = asset.file().getOriginalFilename(); if (filename == null || filename.isBlank()) filename = "terraform.zip"; String path; diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformAssetSupportTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformAssetSupportTest.java index 5074f220..6866ea96 100644 --- a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformAssetSupportTest.java +++ b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformAssetSupportTest.java @@ -73,6 +73,8 @@ void delegatesStoresAndDatabaseQueries() { eq("application/zip"), any(), eq("alice"), eq("127.0.0.1")); verify(hosted).putInternal(eq(runtime), eq("v1/metadata.json"), any(), eq("application/json"), any(), eq("terraform"), eq(null)); + support.delete(runtime, "v1/file.zip"); + verify(hosted).deleteInternal(runtime, "v1/file.zip"); when(assetDao.findAssetByPath(runtime.id(), asset.path())).thenReturn(Optional.of(asset)); when(assetDao.listAssetsByPrefix(runtime.id(), "v1/")).thenReturn(List.of(asset)); diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java index 9746ef71..cbe5d480 100644 --- a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java +++ b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java @@ -33,6 +33,7 @@ import com.github.klboke.kkrepo.server.raw.RawProxyService; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -365,8 +366,11 @@ void rewritesAndVerifiesProxyProviderMetadataAndRoutes() throws Exception { } return response("archive".getBytes(StandardCharsets.UTF_8), "application/zip"); }); + InputStream checksumFailedBody = mock(InputStream.class); when(proxy.getAssetFromUrl(eq(proxyRuntime), anyString(), anyString(), anyBoolean())) - .thenReturn(MavenResponse.noBody(200)); + .thenReturn( + MavenResponse.noBody(200), + MavenResponse.ok(checksumFailedBody, 7, "application/zip", null, null)); Map versions = json(service.get( proxyRuntime, paths.parse("v1/providers/acme/cloud/versions"), BASE, false)); @@ -393,6 +397,8 @@ void rewritesAndVerifiesProxyProviderMetadataAndRoutes() throws Exception { when(assets.blob(archive)).thenReturn(blob(42, "different", 7)); assertThrows(MavenExceptions.BadUpstreamException.class, () -> service.get(proxyRuntime, paths.parse(localArchive), BASE, false)); + verify(checksumFailedBody).close(); + verify(assets).delete(proxyRuntime, localArchive); } @Test diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/upload/ComponentUploadServiceTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/upload/ComponentUploadServiceTest.java index 2f33df4b..d8c37e43 100644 --- a/server/src/test/java/com/github/klboke/kkrepo/server/upload/ComponentUploadServiceTest.java +++ b/server/src/test/java/com/github/klboke/kkrepo/server/upload/ComponentUploadServiceTest.java @@ -23,6 +23,7 @@ import com.github.klboke.kkrepo.server.pypi.PypiHostedService; import com.github.klboke.kkrepo.server.pub.PubHostedService; import com.github.klboke.kkrepo.server.raw.RawHostedService; +import com.github.klboke.kkrepo.server.terraform.TerraformService; import com.github.klboke.kkrepo.server.yum.YumService; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -144,6 +145,33 @@ void pubComponentUploadRejectsNonTarGzAsset() throws Exception { verify(pubHosted, never()).uploadArchive(any(), any(), any(), any(), any()); } + @Test + void terraformComponentUploadRejectsMultipleAssets() throws Exception { + TerraformService terraformService = mock(TerraformService.class); + ComponentUploadService service = service(terraformService); + LinkedMultiValueMap uploads = files("terraform.asset", "provider.zip"); + uploads.add("terraform.asset", new MockMultipartFile( + "terraform.asset", "extra.zip", "application/zip", new byte[] {1})); + + UploadValidationException thrown = assertThrows( + UploadValidationException.class, + () -> service.upload( + "terraform-hosted", + Map.of( + "terraform.kind", new String[] {"provider"}, + "terraform.namespace", new String[] {"acme"}, + "terraform.name", new String[] {"cloud"}, + "terraform.version", new String[] {"1.2.3"}, + "terraform.os", new String[] {"linux"}, + "terraform.arch", new String[] {"amd64"}), + uploads, + "alice", + "127.0.0.1")); + + assertEquals("Terraform upload requires exactly one archive", thrown.getMessage()); + verify(terraformService, never()).put(any(), any(), any(), any(), any(), any(), any()); + } + private static ComponentUploadService service(CargoHostedService cargoHosted) { return service(runtime("cargo-hosted", RepositoryFormat.CARGO), cargoHosted, mock(PubHostedService.class)); } @@ -170,6 +198,25 @@ private static ComponentUploadService service(ComposerHostedService composerHost mock(YumService.class)); } + private static ComponentUploadService service(TerraformService terraformService) { + RepositoryRuntime runtime = runtime("terraform-hosted", RepositoryFormat.TERRAFORM); + RepositoryRuntimeRegistry registry = mock(RepositoryRuntimeRegistry.class); + when(registry.resolve(runtime.name())).thenReturn(Optional.of(runtime)); + return new ComponentUploadService( + registry, + mock(AssetDao.class), + mock(MavenHostedService.class), + mock(NpmHostedService.class), + mock(PypiHostedService.class), + mock(HelmHostedService.class), + mock(CargoHostedService.class), + mock(PubHostedService.class), + mock(ComposerHostedService.class), + mock(RawHostedService.class), + mock(YumService.class), + terraformService); + } + private static ComponentUploadService service( RepositoryRuntime runtime, CargoHostedService cargoHosted, From 2e01c19da5f2e46c84a1adfe6d02bd7ea4723371 Mon Sep 17 00:00:00 2001 From: kl Date: Wed, 15 Jul 2026 12:57:38 +0800 Subject: [PATCH 17/53] fix: propagate Terraform upstream metadata failures --- .../server/terraform/TerraformService.java | 15 +++++- .../terraform/TerraformServiceTest.java | 48 +++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java index 62825511..97bdced7 100644 --- a/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java +++ b/server/src/main/java/com/github/klboke/kkrepo/server/terraform/TerraformService.java @@ -432,7 +432,12 @@ private MavenResponse proxyProviderDownload( Map body = readJson(responseBytes( proxy.getMetadataFromUrl(runtime, cachePath, remote, false))); String filename = string(body.get("filename")); - TerraformPathParser.requireFilename(filename); + try { + TerraformPathParser.requireFilename(filename); + } catch (IllegalArgumentException e) { + throw new MavenExceptions.BadUpstreamException( + "Invalid Terraform upstream provider filename", e); + } String download = absolute(remote, string(body.get("download_url"))); String sums = absolute(remote, string(body.get("shasums_url"))); String signature = absolute(remote, string(body.get("shasums_signature_url"))); @@ -554,6 +559,8 @@ private MavenResponse groupGet( private MavenResponse mergeGroupVersions( RepositoryRuntime group, TerraformPath path, RequestUrls urls, boolean headOnly) { Map> versions = new LinkedHashMap<>(); + MavenExceptions.BadUpstreamException lastUpstreamFailure = null; + boolean producedMetadata = false; for (RepositoryRuntime member : group.members()) { try { Map body = readJson(responseBytes(get(member, path, urls, false))); @@ -575,10 +582,16 @@ private MavenResponse mergeGroupVersions( } } } + producedMetadata = true; + } catch (MavenExceptions.BadUpstreamException e) { + lastUpstreamFailure = e; } catch (RuntimeException ignored) { // Continue with healthy members. } } + if (!producedMetadata && lastUpstreamFailure != null) { + throw lastUpstreamFailure; + } List> sorted = TerraformVersions.descending(versions.keySet()).stream() .map(versions::get).toList(); return path.kind() == TerraformPath.Kind.MODULE_VERSIONS diff --git a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java index cbe5d480..74ddbfc3 100644 --- a/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java +++ b/server/src/test/java/com/github/klboke/kkrepo/server/terraform/TerraformServiceTest.java @@ -547,6 +547,54 @@ void rejectsMalformedProxyProviderMetadata() throws Exception { verify(signatureVerifier, never()).verify(any(), any(), any()); } + @Test + void wrapsInvalidProxyProviderFilenameAsBadUpstream() throws Exception { + RepositoryRuntime proxyRuntime = runtime( + 72, "terraform-proxy", RepositoryType.PROXY, "https://registry.example", List.of()); + when(proxy.getMetadataFromUrl(eq(proxyRuntime), anyString(), anyString(), anyBoolean())) + .thenAnswer(invocation -> { + String local = invocation.getArgument(1); + if (local.startsWith(".terraform/upstream/discovery-")) { + return response(mapper.writeValueAsBytes(Map.of("providers.v1", "/v1/providers/")), + "application/json"); + } + return response(mapper.writeValueAsBytes(Map.of("filename", "../provider.zip")), + "application/json"); + }); + + MavenExceptions.BadUpstreamException thrown = assertThrows( + MavenExceptions.BadUpstreamException.class, + () -> service.get(proxyRuntime, + paths.parse("v1/providers/acme/cloud/1.2.3/download/linux/amd64"), BASE, false)); + + assertTrue(thrown.getMessage().contains("provider filename")); + assertTrue(thrown.getCause() instanceof IllegalArgumentException); + } + + @Test + void propagatesGroupUpstreamFailureWhenNoMemberProducesVersionMetadata() throws Exception { + RepositoryRuntime unavailable = runtime( + 73, "terraform-proxy", RepositoryType.PROXY, "https://registry.example", List.of()); + RepositoryRuntime group = runtime( + 74, "terraform-group", RepositoryType.GROUP, null, List.of(unavailable)); + when(proxy.getMetadataFromUrl(eq(unavailable), anyString(), anyString(), anyBoolean())) + .thenAnswer(invocation -> { + String local = invocation.getArgument(1); + if (local.startsWith(".terraform/upstream/discovery-")) { + return response(mapper.writeValueAsBytes(Map.of("providers.v1", "/v1/providers/")), + "application/json"); + } + throw new MavenExceptions.BadUpstreamException("registry unavailable"); + }); + + MavenExceptions.BadUpstreamException thrown = assertThrows( + MavenExceptions.BadUpstreamException.class, + () -> service.get( + group, paths.parse("v1/providers/acme/cloud/versions"), BASE, false)); + + assertEquals("registry unavailable", thrown.getMessage()); + } + @Test void keysDiscoveryMetadataByConfiguredRemoteUrl() throws Exception { RepositoryRuntime original = runtime( From ff6097e58262167843e43929ed6e931c9c926e69 Mon Sep 17 00:00:00 2001 From: kl Date: Wed, 15 Jul 2026 13:18:58 +0800 Subject: [PATCH 18/53] fix: complete Terraform browse uploads --- .../resources/browse/assets/browse.js | 108 ++++++++++++++++++ .../META-INF/resources/browse/index.html | 2 +- .../ui/BrowseTerraformUploadContractTest.java | 39 +++++++ .../terraform/TerraformPathParser.java | 2 +- .../terraform/TerraformPathParserTest.java | 4 + .../server/terraform/TerraformService.java | 8 +- .../terraform/TerraformServiceTest.java | 4 + 7 files changed, 164 insertions(+), 3 deletions(-) create mode 100644 browse-ui/src/test/java/com/github/klboke/kkrepo/browse/ui/BrowseTerraformUploadContractTest.java diff --git a/browse-ui/src/main/resources/META-INF/resources/browse/assets/browse.js b/browse-ui/src/main/resources/META-INF/resources/browse/assets/browse.js index f0a1131a..5c891174 100644 --- a/browse-ui/src/main/resources/META-INF/resources/browse/assets/browse.js +++ b/browse-ui/src/main/resources/META-INF/resources/browse/assets/browse.js @@ -2610,6 +2610,53 @@ function renderUploadFields() { bindRemoveAssetButtons(); return; } + if (repo.format === "terraform") { + fields.innerHTML = ` + + + + + + + + + + `; + document.getElementById("upload-terraform-kind") + .addEventListener("change", updateTerraformUploadKind); + updateTerraformUploadKind(); + return; + } fields.innerHTML = ` +