diff --git a/Cargo.lock b/Cargo.lock index d61ee233ca..16d855dcfc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1169,6 +1169,7 @@ dependencies = [ "carbide-ib-partition-controller", "carbide-ipmi", "carbide-ipxe-renderer", + "carbide-kms-provider", "carbide-libmlx", "carbide-machine-controller", "carbide-macros", @@ -1280,7 +1281,9 @@ dependencies = [ "tss-esapi", "url", "uuid", + "vaultrs", "x509-parser", + "zeroize", ] [[package]] @@ -1338,8 +1341,12 @@ name = "carbide-api-integration-tests" version = "0.0.0" dependencies = [ "askama_escape", + "base64", "bmc-mock", + "carbide-api-core", + "carbide-api-db", "carbide-api-test-helper", + "carbide-kms-provider", "carbide-machine-a-tron", "carbide-secrets", "carbide-utils", @@ -2151,6 +2158,7 @@ dependencies = [ "tempfile", "thiserror 2.0.18", "tokio", + "tokio-util", "tracing", "vaultrs", "zeroize", diff --git a/crates/admin-cli/cli_domains.yaml b/crates/admin-cli/cli_domains.yaml index 396bc26010..048aba0a85 100644 --- a/crates/admin-cli/cli_domains.yaml +++ b/crates/admin-cli/cli_domains.yaml @@ -81,3 +81,4 @@ admin: - dev-env - ssh - jump + - secrets diff --git a/crates/admin-cli/src/cfg/cli_options.rs b/crates/admin-cli/src/cfg/cli_options.rs index 3df4ffc47c..268c8ed495 100644 --- a/crates/admin-cli/src/cfg/cli_options.rs +++ b/crates/admin-cli/src/cfg/cli_options.rs @@ -25,9 +25,9 @@ use crate::{ ipxe_template, jump, machine, machine_interfaces, machine_validation, managed_host, managed_switch, mlx, network_devices, network_security_group, network_segment, nvl_domain, nvl_logical_partition, nvl_partition, nvlink_nmxc_endpoints, operating_system, os_image, ping, - power_shelf, rack, redfish, resource_pool, rms, route_server, scout_stream, set, site_explorer, - sku, spx_partition, ssh, switch, tenant, tenant_keyset, tpm_ca, trim_table, version, vpc, - vpc_peering, vpc_prefix, + power_shelf, rack, redfish, resource_pool, rms, route_server, scout_stream, secrets, set, + site_explorer, sku, spx_partition, ssh, switch, tenant, tenant_keyset, tpm_ca, trim_table, + version, vpc, vpc_peering, vpc_prefix, }; #[derive(Parser, Debug)] @@ -202,6 +202,8 @@ pub enum CliCommand { ExtensionService(extension_service::Cmd), #[clap(about = "Firmware related actions", subcommand)] Firmware(firmware::Cmd), + #[clap(about = "Secrets management", subcommand)] + Secrets(secrets::Cmd), #[clap( about = "Regenerate the docs/manuals/nico-admin-cli markdown reference", hide = true diff --git a/crates/admin-cli/src/main.rs b/crates/admin-cli/src/main.rs index a5234ff7dd..f66d6e8a7e 100644 --- a/crates/admin-cli/src/main.rs +++ b/crates/admin-cli/src/main.rs @@ -103,6 +103,7 @@ mod rms; mod route_server; mod rpc; mod scout_stream; +mod secrets; mod set; mod site_explorer; mod sku; @@ -274,6 +275,7 @@ async fn main() -> color_eyre::Result<()> { CliCommand::ResourcePool(cmd) => cmd.dispatch(ctx).await?, CliCommand::RouteServer(cmd) => cmd.dispatch(ctx).await?, CliCommand::ScoutStream(cmd) => cmd.dispatch(ctx).await?, + CliCommand::Secrets(cmd) => cmd.dispatch(ctx).await?, CliCommand::Set(cmd) => cmd.dispatch(ctx).await?, CliCommand::Ssh(cmd) => cmd.dispatch(ctx).await?, CliCommand::SiteExplorer(cmd) => cmd.dispatch(ctx).await?, diff --git a/crates/admin-cli/src/secrets/mod.rs b/crates/admin-cli/src/secrets/mod.rs new file mode 100644 index 0000000000..9b4216817a --- /dev/null +++ b/crates/admin-cli/src/secrets/mod.rs @@ -0,0 +1,31 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +mod re_wrap; + +use clap::Parser; + +use crate::cfg::dispatch::Dispatch; + +#[derive(Parser, Debug, Clone, Dispatch)] +#[clap(rename_all = "kebab_case")] +pub enum Cmd { + #[clap(about = "Re-wrap secret DEKs to use the \ + currently active KEK per routing \ + config")] + ReWrap(re_wrap::Args), +} diff --git a/crates/admin-cli/src/secrets/re_wrap/args.rs b/crates/admin-cli/src/secrets/re_wrap/args.rs new file mode 100644 index 0000000000..257d16407d --- /dev/null +++ b/crates/admin-cli/src/secrets/re_wrap/args.rs @@ -0,0 +1,38 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use clap::Parser; + +#[derive(Parser, Debug, Clone)] +#[command(after_long_help = "\ +EXAMPLES: + +Re-wrap every credential whose KEK no longer matches the routing config +(run this after rotating a key in [secrets.routing]): + $ nico-admin-cli secrets re-wrap + +Use a smaller batch size to lighten load on an external KMS: + $ nico-admin-cli secrets re-wrap --batch-size 25 + +")] +pub struct Args { + #[clap( + long, + help = "Rows scanned per batch during the walk. The server applies its own default and limits." + )] + pub batch_size: Option, +} diff --git a/crates/admin-cli/src/secrets/re_wrap/cmd.rs b/crates/admin-cli/src/secrets/re_wrap/cmd.rs new file mode 100644 index 0000000000..282eeb3e83 --- /dev/null +++ b/crates/admin-cli/src/secrets/re_wrap/cmd.rs @@ -0,0 +1,43 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use crate::errors::CarbideCliResult; + +use crate::rpc::ApiClient; + +pub async fn re_wrap(api_client: &ApiClient, batch_size: Option) -> CarbideCliResult<()> { + let request = ::rpc::forge::ReWrapSecretsRequest { batch_size }; + + let resp = api_client.0.re_wrap_secrets(request).await?; + + println!( + "Re-wrap complete: {} re-wrapped, {} already current", + resp.re_wrapped, resp.already_current + ); + if resp.stale_remaining == 0 { + println!( + "No rows remain on KEKs outside the routing config; unrouted KEKs can be retired." + ); + } else { + println!( + "{} rows are still wrapped by KEKs outside the routing config -- \ + concurrent writers likely landed rows mid-walk; run re-wrap again.", + resp.stale_remaining + ); + } + Ok(()) +} diff --git a/crates/admin-cli/src/secrets/re_wrap/mod.rs b/crates/admin-cli/src/secrets/re_wrap/mod.rs new file mode 100644 index 0000000000..2ad7de3a67 --- /dev/null +++ b/crates/admin-cli/src/secrets/re_wrap/mod.rs @@ -0,0 +1,31 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +pub mod args; +mod cmd; + +use crate::errors::CarbideCliResult; +pub use args::Args; + +use crate::cfg::run::Run; +use crate::cfg::runtime::RuntimeContext; + +impl Run for Args { + async fn run(self, ctx: &mut RuntimeContext) -> CarbideCliResult<()> { + cmd::re_wrap(&ctx.api_client, self.batch_size).await + } +} diff --git a/crates/api-core/Cargo.toml b/crates/api-core/Cargo.toml index f2b1378a11..4ec0563ad2 100644 --- a/crates/api-core/Cargo.toml +++ b/crates/api-core/Cargo.toml @@ -42,6 +42,7 @@ carbide-ib-fabric = { path = "../ib-fabric" } carbide-ib-partition-controller = { path = "../ib-partition-controller" } carbide-ipmi = { path = "../ipmi" } carbide-ipxe-renderer = { path = "../ipxe-renderer" } +carbide-kms-provider = { path = "../kms-provider" } carbide-libmlx = { path = "../libmlx" } carbide-machine-controller = { path = "../machine-controller" } carbide-measured-boot = { path = "../measured-boot", features = ["sqlx"] } @@ -180,7 +181,9 @@ tracing-subscriber = { workspace = true, features = [ tss-esapi = { workspace = true, optional = true } url = { workspace = true, features = ["serde"] } uuid = { workspace = true, features = ["v4", "serde"] } +vaultrs = { workspace = true } x509-parser = { workspace = true, features = ["verify"] } +zeroize = { workspace = true } [features] default = ["linux-build"] diff --git a/crates/api-core/src/api.rs b/crates/api-core/src/api.rs index dd5be74faf..a9d87838f1 100644 --- a/crates/api-core/src/api.rs +++ b/crates/api-core/src/api.rs @@ -88,6 +88,7 @@ pub struct Api { pub(crate) metric_emitter: ApiMetricsEmitter, pub(crate) component_manager: Option, pub(crate) bms_client: OnceLock>, + pub(crate) secrets_context: Option, } pub(crate) type ScoutStreamType = @@ -1444,6 +1445,13 @@ impl Forge for Api { crate::handlers::credential::delete_credential(self, request).await } + async fn re_wrap_secrets( + &self, + request: Request, + ) -> Result, Status> { + crate::handlers::secrets::re_wrap_secrets(self, request).await + } + /// get_route_servers returns a list of all configured route server /// entries for all source types. async fn get_route_servers( diff --git a/crates/api-core/src/auth/internal_rbac_rules.rs b/crates/api-core/src/auth/internal_rbac_rules.rs index 92c6527f83..c0124b5308 100644 --- a/crates/api-core/src/auth/internal_rbac_rules.rs +++ b/crates/api-core/src/auth/internal_rbac_rules.rs @@ -463,6 +463,7 @@ impl InternalRBACRules { "UpdateOperatingSystemCachableIpxeTemplateArtifacts", vec![ForgeAdminCLI], ); + x.perm("ReWrapSecrets", vec![ForgeAdminCLI]); x.perm("GetIpxeTemplate", vec![ForgeAdminCLI, SiteAgent]); x.perm("ListIpxeTemplates", vec![ForgeAdminCLI, SiteAgent]); x.perm("FindRackStateHistories", vec![ForgeAdminCLI, Machineatron]); diff --git a/crates/api-core/src/cfg/file.rs b/crates/api-core/src/cfg/file.rs index cda2620437..f38d58e4ef 100644 --- a/crates/api-core/src/cfg/file.rs +++ b/crates/api-core/src/cfg/file.rs @@ -725,6 +725,11 @@ pub struct CarbideConfig { #[serde(default)] pub tracing: TracingConfig, + + /// Secrets backend configuration. When present, credentials live + /// encrypted in Postgres and vault leaves the credential chain + /// entirely; when absent, vault remains the credential store. + pub secrets: Option, } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -837,6 +842,102 @@ pub enum BgpLeafSessionPassword { SiteWide, } +/// Configures the Postgres secrets backend. When this section is present, +/// credentials live encrypted in Postgres and vault is not in the +/// credential chain at all -- the one-time import either completes before +/// the process serves traffic, or the process does not start. Vault keeps +/// serving PKI certificates either way. +/// +/// Enabling this on an existing site has two prerequisites that live +/// outside this process: +/// +/// - Services that read credentials from vault through their own chains +/// (`bmc-proxy`, `dsx-exchange-consumer`) keep reading vault and will +/// not see anything carbide-api writes to Postgres afterwards. They must +/// be migrated or fed another way before credentials here change. +/// - During a rolling upgrade, replicas still running the vault config +/// keep writing rotated credentials to vault, where they are stranded +/// once the import has completed. Keep autonomous credential writers +/// (site-explorer credential rotation) disabled until the whole fleet +/// runs this config. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct SecretsConfig { + /// KMS backend configuration. + pub kms: KmsConfig, + + /// Maps path prefixes to the kek_id that encrypts new writes under + /// them, longest prefix winning. A "/" catch-all entry is required. + /// Reads never consult routing -- every stored row records the KEK + /// that wrapped it -- so rotating a key means changing it here and + /// running `carbide-admin-cli secrets re-wrap`. + /// + /// Example: + /// ```toml + /// [secrets.routing] + /// "/" = "default-key" + /// "machines/bmc" = "bmc-key" + /// ``` + pub routing: std::collections::HashMap, + + /// A source backend to import secrets from at startup. Unset means a + /// fresh site with nothing to import; unsupported values fail config + /// parsing rather than silently skipping the import. + pub import_from: Option, + + /// How to treat secrets that already exist in Postgres during import. + /// Defaults to missing_only. + #[serde(default)] + pub import_approach: crate::secrets::ImportApproach, +} + +/// A backend the one-time secrets import can read from. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ImportSource { + Vault, +} + +/// Configures the KMS backends that wrap DEKs. Several named providers can +/// be defined: the active one wraps DEKs for new writes, and every provider +/// answers unwraps for the kek_ids it has. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct KmsConfig { + /// The provider that wraps DEKs for new writes. + pub active: String, + + /// Named provider configurations. + pub providers: std::collections::HashMap, +} + +/// One KMS provider. The `type` field in TOML selects the variant, and each +/// variant only accepts the fields that belong to it -- an integrated +/// provider cannot be given a transit key list, a misspelled field is a +/// parse error, and so on. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] +pub enum ProviderConfig { + /// Local key material, loaded from the environment or files. The + /// default backend when no external KMS exists. + Integrated { + /// kek_id to key source. Key material itself never appears in + /// this config -- only where to find it. + keys: std::collections::HashMap, + }, + /// Vault/OpenBao Transit, which wraps and unwraps DEKs server-side. + /// Requires a static vault token in the credential config -- the + /// Kubernetes service-account login flow is not supported for transit + /// yet. + Transit { + /// The Transit key names this provider answers for. + keys: Vec, + /// The Transit secrets engine mount path. Defaults to "transit". + #[serde(default)] + transit_mount: Option, + }, +} + #[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] #[serde(rename_all = "snake_case")] pub enum ComputeAllocationEnforcement { @@ -3982,4 +4083,185 @@ firmware_url = "https://firmware.example.com/fw-b.bin" DEFAULT_DPF_IMAGE_PULL_SECRET ); } + + // Verifies that a [secrets] config section with KMS, routing, and import settings + // deserializes correctly from TOML. + #[test] + fn secrets_config_deserializes_from_toml() { + #[derive(Deserialize)] + struct Wrapper { + secrets: SecretsConfig, + } + + let toml_str = r#" + [secrets] + import_from = "vault" + import_approach = "missing_only" + + [secrets.kms] + active = "local" + + [secrets.kms.providers.local] + type = "integrated" + keys.default-key = { env = "CARBIDE_SECRETS_KEY_DEFAULT" } + keys.bmc-key = { file = "/run/secrets/bmc-key" } + + [secrets.kms.providers.prod-transit] + type = "transit" + keys = ["my-transit-key"] + + [secrets.routing] + "/" = "default-key" + "machines/bmc" = "bmc-key" + "#; + + let wrapper: Wrapper = toml::from_str(toml_str).expect("parse secrets config"); + let secrets = wrapper.secrets; + + // Verify KMS config: the `type` field selects the enum variant. + assert_eq!(secrets.kms.active, "local"); + assert_eq!(secrets.kms.providers.len(), 2); + assert!(matches!( + &secrets.kms.providers["local"], + ProviderConfig::Integrated { keys } if keys.len() == 2 + )); + assert!(matches!( + &secrets.kms.providers["prod-transit"], + ProviderConfig::Transit { keys, transit_mount: None } if keys == &["my-transit-key"] + )); + + // Verify routing. + assert_eq!(secrets.routing.len(), 2); + assert_eq!(secrets.routing["/"], "default-key"); + assert_eq!(secrets.routing["machines/bmc"], "bmc-key"); + + // Verify import settings. + assert_eq!(secrets.import_from, Some(ImportSource::Vault)); + assert_eq!( + secrets.import_approach, + crate::secrets::ImportApproach::MissingOnly + ); + } + + // Verifies that a typo'd import source fails config parsing instead of + // silently skipping the import. + #[test] + fn secrets_config_rejects_unknown_import_source() { + #[derive(Deserialize)] + struct Wrapper { + #[expect(dead_code)] + secrets: SecretsConfig, + } + + let toml_str = r#" + [secrets] + import_from = "valt" + + [secrets.kms] + active = "local" + + [secrets.kms.providers.local] + type = "integrated" + keys.default-key = { env = "CARBIDE_SECRETS_KEY_DEFAULT" } + + [secrets.routing] + "/" = "default-key" + "#; + + assert!(toml::from_str::(toml_str).is_err()); + } + + // Verifies that a misspelled optional key in [secrets] -- here + // `import_fom` for `import_from` -- fails to parse instead of leaving + // the import silently disabled. Without deny_unknown_fields, the typo'd + // key is ignored and an existing site can boot on empty Postgres. + #[test] + fn secrets_config_rejects_misspelled_field() { + #[derive(Deserialize)] + struct Wrapper { + #[expect(dead_code)] + secrets: SecretsConfig, + } + + let toml_str = r#" + [secrets] + import_fom = "vault" + + [secrets.kms] + active = "local" + + [secrets.kms.providers.local] + type = "integrated" + keys.default-key = { env = "CARBIDE_SECRETS_KEY_DEFAULT" } + + [secrets.routing] + "/" = "default-key" + "#; + + assert!(toml::from_str::(toml_str).is_err()); + } + + // Verifies that a field belonging to the other provider type -- here + // transit_mount on an integrated provider -- fails to parse instead of + // being silently ignored. + #[test] + fn secrets_config_rejects_unknown_provider_field() { + #[derive(Deserialize)] + struct Wrapper { + #[expect(dead_code)] + secrets: SecretsConfig, + } + + let toml_str = r#" + [secrets.kms] + active = "local" + + [secrets.kms.providers.local] + type = "integrated" + transit_mount = "transit" + keys.default-key = { env = "CARBIDE_SECRETS_KEY_DEFAULT" } + + [secrets.routing] + "/" = "default-key" + "#; + + assert!(toml::from_str::(toml_str).is_err()); + } + + // Verifies that a provider with the wrong field for its type -- here an + // integrated provider given transit's key list -- fails to parse + // instead of deferring the mistake to startup. + #[test] + fn secrets_config_rejects_mismatched_provider_fields() { + #[derive(Deserialize)] + struct Wrapper { + #[expect(dead_code)] + secrets: SecretsConfig, + } + + let toml_str = r#" + [secrets.kms] + active = "local" + + [secrets.kms.providers.local] + type = "integrated" + keys = ["not-a-key-map"] + + [secrets.routing] + "/" = "default-key" + "#; + + assert!(toml::from_str::(toml_str).is_err()); + } + + // Verifies that secrets config is optional — a config without [secrets] should have None. + #[test] + fn secrets_config_absent_by_default() { + let config: CarbideConfig = Figment::new() + .merge(Toml::file(format!("{TEST_DATA_DIR}/min_config.toml"))) + .extract() + .unwrap(); + + assert!(config.secrets.is_none()); + } } diff --git a/crates/api-core/src/handlers/mod.rs b/crates/api-core/src/handlers/mod.rs index 88aa3c38a6..f3ee50aca2 100644 --- a/crates/api-core/src/handlers/mod.rs +++ b/crates/api-core/src/handlers/mod.rs @@ -73,6 +73,7 @@ pub mod redfish; pub mod resource_pool; pub mod route_server; pub mod scout_stream; +pub mod secrets; pub mod site_explorer; pub mod sku; pub mod spx_partition; diff --git a/crates/api-core/src/handlers/secrets.rs b/crates/api-core/src/handlers/secrets.rs new file mode 100644 index 0000000000..5f801ac5f3 --- /dev/null +++ b/crates/api-core/src/handlers/secrets.rs @@ -0,0 +1,79 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use tonic::{Request, Response, Status}; + +use crate::CarbideError; +use crate::api::Api; + +const DEFAULT_RE_WRAP_BATCH_SIZE: i64 = 100; + +/// Re-wrap batches stay within this range: zero would scan nothing while +/// reporting success, and an enormous batch holds that many rows -- and +/// their sequential KMS round-trips -- in memory between commits. +const RE_WRAP_BATCH_SIZE_RANGE: std::ops::RangeInclusive = 1..=10_000; + +pub(crate) async fn re_wrap_secrets( + api: &Api, + request: Request, +) -> Result, Status> { + crate::api::log_request_data(&request); + + let req = request.into_inner(); + let batch_size = req + .batch_size + .map(|b| { + i64::from(b).clamp( + *RE_WRAP_BATCH_SIZE_RANGE.start(), + *RE_WRAP_BATCH_SIZE_RANGE.end(), + ) + }) + .unwrap_or(DEFAULT_RE_WRAP_BATCH_SIZE); + + let ctx = api.secrets_context.as_ref().ok_or_else(|| { + CarbideError::FailedPrecondition( + "secrets backend not configured -- no [secrets] section in config".to_string(), + ) + })?; + + let result = crate::secrets::re_wrap_stale( + &api.database_connection, + ctx.kms.as_ref(), + &ctx.routing, + batch_size, + ) + .await + .map_err(|e| match e { + crate::secrets::PgSecretsError::ReWrapInProgress => { + CarbideError::FailedPrecondition(e.to_string()) + } + other => CarbideError::internal(format!("re-wrap failed: {other}")), + })?; + + tracing::info!( + re_wrapped = result.re_wrapped, + already_current = result.already_current, + stale_remaining = result.stale_remaining, + "secrets re-wrap completed" + ); + + Ok(Response::new(rpc::forge::ReWrapSecretsResponse { + re_wrapped: result.re_wrapped, + already_current: result.already_current, + stale_remaining: result.stale_remaining, + })) +} diff --git a/crates/api-core/src/lib.rs b/crates/api-core/src/lib.rs index fd830d279a..6af5963f6f 100644 --- a/crates/api-core/src/lib.rs +++ b/crates/api-core/src/lib.rs @@ -68,6 +68,7 @@ mod mqtt_state_change_hook; mod network_segment; mod run; mod scout_stream; +pub mod secrets; pub mod setup; mod storage; diff --git a/crates/api-core/src/run.rs b/crates/api-core/src/run.rs index 7166eb2a18..c88a0e92e7 100644 --- a/crates/api-core/src/run.rs +++ b/crates/api-core/src/run.rs @@ -14,27 +14,33 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +use std::collections::BTreeMap; use std::path::PathBuf; use std::sync::Arc; -use carbide_secrets::credentials::{CredentialManager, CredentialReader}; +use carbide_kms_provider::{ + DEFAULT_TRANSIT_MOUNT, IntegratedKmsProvider, KmsBackend, MultiKmsProvider, TransitKmsProvider, +}; +use carbide_secrets::credentials::{CredentialManager, CredentialReader, CredentialWriter}; use carbide_secrets::{ - CredentialConfig, MemoryCredentialStore, VaultConfig, create_credential_manager_from, - create_vault_client, + CredentialConfig, ForgeVaultClient, MemoryCredentialStore, VaultConfig, + create_credential_manager_from, create_vault_client, }; use carbide_utils::HostPortPair; use eyre::WrapErr; +use sqlx::PgPool; use tokio::sync::oneshot::Sender; use tokio::task::JoinSet; use tokio_util::sync::CancellationToken; use tracing::subscriber::NoSubscriber; -use crate::cfg::file::CarbideConfig; +use crate::cfg::file::{CarbideConfig, ImportSource, ProviderConfig, SecretsConfig}; use crate::listener::AdminUiRoutesBuilder; use crate::logging::metrics_endpoint::{MetricsEndpointConfig, run_metrics_endpoint}; use crate::logging::setup::{ Logging, create_metric_for_spancount_reader, create_metrics, setup_logging, }; +use crate::secrets::{SecretRouting, SecretsContext}; use crate::{CarbideError, dynamic_settings, setup}; /// Vault machine PKI URI SANs must match `[auth.trust]` when site auth config is present. @@ -187,44 +193,21 @@ pub async fn run( let vault_config = vault_config_for_site(&credential_config.vault, &carbide_config); - let certificate_provider = create_vault_client(&vault_config, metrics.meter.clone())?; + // One vault client serves every vault role below. PKI certificates stay + // on vault no matter which credential backend is configured. + let vault_client = create_vault_client(&vault_config, metrics.meter.clone())?; + let certificate_provider = vault_client.clone(); - // Pick a credential store based on CARBIDE_CREDENTIAL_STORE (default: "vault"). - // Set to "memory" to use an in-memory store with no persistence or shared state between - // processes. This is only suitable for development and testing. - let credential_store: Arc = match std::env::var( - "CARBIDE_CREDENTIAL_STORE", - ) - .as_deref() - .unwrap_or("vault") - { - "vault" => create_vault_client(&vault_config, metrics.meter.clone())?, - "memory" => Arc::new(MemoryCredentialStore::default()), - other => { - return Err(eyre::eyre!( - "Invalid CARBIDE_CREDENTIAL_STORE value {other:?}: expected \"vault\" or \"memory\"" - )); - } - }; + let db_pool = setup::create_and_connect_postgres_pool(&carbide_config).await?; - // Build credential reader chain. The idea is this chain - // can be flexible, to allow us to introduce an ordered - // list of readers, which we build on-demand based on - // configuration. + // Build the credential reader chain. Lookups try each reader in order + // and the first answer wins. let mut readers: Vec> = Vec::new(); - - // If EnvCredentials are enabled, then add that - // to our chained credentials reader. It's expected - // that this comes first if configured. if credential_config.env.enabled() { readers.push(Box::new( carbide_secrets::local_credentials::EnvCredentials::new(credential_config.env.clone())?, )); } - - // Next, if FileCredentials are enabled, then - // add those in as well. We expect these *after* - // EnvCredentials. if credential_config.file.enabled() { readers.push(Box::new( carbide_secrets::local_credentials::FileCredentialsWatcher::new( @@ -234,12 +217,69 @@ pub async fn run( )); } - // Last, we tack on the credential store to the end. - readers.push(Box::new(credential_store.clone())); + // With a [secrets] section, Postgres is the whole credential backend: + // it answers reads, takes every write, and vault is not in the chain + // at all -- the strict one-time import below either completes or the + // process does not start. Without the section, the store comes from + // CARBIDE_CREDENTIAL_STORE: vault (the default), or an in-memory store + // suitable only for development and testing. + let (writer, secrets_context): (Arc, Option) = + if let Some(ref secrets_config) = carbide_config.secrets { + let routing = SecretRouting::from_config(&secrets_config.routing) + .map_err(eyre::Report::new) + .wrap_err("secrets routing configuration")?; + let kms = build_kms_backend( + secrets_config, + &vault_config, + &routing, + &mut join_set, + &cancel_token, + )?; + + let pg_mgr = Arc::new( + crate::secrets::PostgresCredentialManager::new( + db_pool.clone(), + routing.clone(), + kms.clone(), + ) + .with_metrics(crate::secrets::SecretsMetrics::new(&metrics.meter)), + ); + tracing::info!( + active_provider = %secrets_config.kms.active, + "Postgres secrets backend configured" + ); + + import_vault_secrets_once( + &db_pool, + secrets_config, + &routing, + kms.as_ref(), + &vault_client, + ) + .await?; + + readers.push(Box::new(pg_mgr.clone()) as Box); + (pg_mgr, Some(SecretsContext { routing, kms })) + } else { + let credential_store: Arc = match std::env::var( + "CARBIDE_CREDENTIAL_STORE", + ) + .as_deref() + .unwrap_or("vault") + { + "vault" => vault_client.clone(), + "memory" => Arc::new(MemoryCredentialStore::default()), + other => { + return Err(eyre::eyre!( + "Invalid CARBIDE_CREDENTIAL_STORE value {other:?}: expected \"vault\" or \"memory\"" + )); + } + }; + readers.push(Box::new(credential_store.clone())); + (credential_store, None) + }; - // And now we create our new composite credential manager - // from the list of readers we just built, plus the credential store as writer. - let credential_manager = create_credential_manager_from(credential_store, readers); + let credential_manager = create_credential_manager_from(writer, readers); let redfish_pool = { let rf_pool = libredfish::RedfishClientPool::builder() @@ -310,6 +350,8 @@ pub async fn run( nv_redfish_pool, credential_manager, certificate_provider, + db_pool, + secrets_context, admin_ui_routes_builder, cancel_token, ready_channel, @@ -322,3 +364,231 @@ pub async fn run( Ok(()) } + +/// Build the KMS stack from the `[secrets.kms]` config: construct every +/// named provider, check the routed KEKs against them, and combine them so +/// the active provider wraps DEKs for new writes while any provider can +/// unwrap rows recorded with its kek_ids. +fn build_kms_backend( + secrets_config: &SecretsConfig, + vault_config: &VaultConfig, + routing: &SecretRouting, + join_set: &mut JoinSet<()>, + cancel_token: &CancellationToken, +) -> eyre::Result> { + // BTreeMap so the provider list below has a stable order -- with + // duplicate kek_ids rejected, order never decides which provider + // unwraps, but stable beats arbitrary if that invariant ever slips. + let mut built: BTreeMap> = BTreeMap::new(); + + for (name, provider_config) in &secrets_config.kms.providers { + let provider: Arc = match provider_config { + ProviderConfig::Integrated { keys } => Arc::new( + IntegratedKmsProvider::from_config(keys) + .map_err(eyre::Report::new) + .wrap_err_with(|| format!("KMS provider {name:?} key configuration"))?, + ), + ProviderConfig::Transit { + keys, + transit_mount, + } => { + // The same address, CA trust, and timeout ForgeVaultClient + // connects with -- a bare vaultrs client only trusts public + // roots and fails TLS against a site-CA-signed vault. + let vault_settings = + carbide_secrets::create_raw_vault_client_settings(vault_config).wrap_err( + "building the Transit KMS vault client (Transit requires a static \ + VAULT_TOKEN; the Kubernetes service-account login flow is not \ + supported for Transit yet)", + )?; + let vault_client = Arc::new( + vaultrs::client::VaultClient::new(vault_settings) + .map_err(|e| eyre::eyre!("vault client: {e}"))?, + ); + let transit_provider = TransitKmsProvider::new( + vault_client, + transit_mount + .as_deref() + .unwrap_or(DEFAULT_TRANSIT_MOUNT) + .to_string(), + keys.clone(), + ); + join_set + .build_task() + .name("transit_kms_token_renewal") + .spawn(transit_provider.run_token_renewal(cancel_token.clone()))?; + Arc::new(transit_provider) + } + }; + tracing::info!(name = %name, "initialized KMS provider"); + built.insert(name.clone(), provider); + } + + let active = built + .get(&secrets_config.kms.active) + .ok_or_else(|| { + eyre::eyre!( + "active KMS provider {:?} not found; configured providers: {:?}", + secrets_config.kms.active, + built.keys().collect::>() + ) + })? + .clone(); + + // Check the config against itself now, while a mismatch is a config + // mistake. Found at runtime instead, a missing key is a write failure + // on whichever credential first routes to it, and a duplicated key + // makes unwraps depend on provider order. + // + // Every routed KEK must exist in the active provider, because all new + // DEK wraps go through it. And no KEK may exist in two providers -- + // checked across every configured KEK, not just the routed ones, + // because rows wrapped by a rotated-out KEK still unwrap through + // whichever provider has it. + for (prefix, kek_id) in routing.routes() { + if !active.can_decrypt_kek(kek_id) { + return Err(eyre::eyre!( + "routing assigns {kek_id:?} (prefix {prefix:?}), but the active KMS \ + provider {:?} does not have that key", + secrets_config.kms.active + )); + } + } + let mut kek_owners: BTreeMap> = BTreeMap::new(); + for (name, provider) in &built { + // Dedup within a provider first: a transit key list can repeat a + // name, and that is harmless, not "two providers". + let mut kek_ids = provider.kek_ids(); + kek_ids.sort(); + kek_ids.dedup(); + for kek_id in kek_ids { + kek_owners.entry(kek_id).or_default().push(name); + } + } + for (kek_id, owners) in &kek_owners { + if owners.len() > 1 { + return Err(eyre::eyre!( + "kek_id {kek_id:?} exists in more than one KMS provider \ + ({owners:?}); unwraps would be ambiguous" + )); + } + } + + let providers: Vec> = built.into_values().collect(); + Ok(Arc::new(MultiKmsProvider::new(active, providers))) +} + +/// Run the one-time vault import if the config asks for one and the +/// completion marker is not written yet. +/// +/// The import either completes before this process serves traffic, or the +/// process does not start: enumeration is strict (any vault list or read +/// failure aborts the boot), and an empty enumeration aborts too, because +/// an empty vault on a site configured to import from it is far more +/// likely a vault problem than a truly empty vault. A genuinely fresh +/// site simply omits `import_from`. Keeping it this absolute is what lets +/// the credential chain exclude vault entirely whenever `[secrets]` is +/// active -- there is no degraded half-migrated state to reason about. +/// +/// Rolling upgrades still need care: a replica running the old vault +/// config can write credentials to vault after this import completes, and +/// those writes are stranded there. Site-explorer credential rotation is +/// the writer to worry about; keep it disabled until the whole fleet runs +/// the `[secrets]` config. +async fn import_vault_secrets_once( + db_pool: &PgPool, + secrets_config: &SecretsConfig, + routing: &SecretRouting, + kms: &dyn KmsBackend, + vault_client: &ForgeVaultClient, +) -> eyre::Result<()> { + if secrets_config.import_from != Some(ImportSource::Vault) { + return Ok(()); + } + + if is_import_complete(db_pool).await? { + tracing::info!("Vault import already completed"); + return Ok(()); + } + + // Several replicas can boot against the same empty database at once. + // The marker path's advisory lock lets one of them import while the + // rest wait here, re-check the marker, and move on. It is a session + // lock on a dedicated connection rather than a transaction-scoped one: + // the import awaits Vault enumeration and pool-backed writes, and + // holding a transaction across those would trip `txn_held_across_await` + // and, under concurrent startup, risk waiters starving the pool the + // importer itself needs. Detaching the connection guarantees the lock + // releases when it drops, including on an early error return. + let mut lock_conn = db_pool + .acquire() + .await + .wrap_err("acquire vault import lock connection")? + .detach(); + db::secrets::lock_path_session(&mut lock_conn, crate::secrets::VAULT_IMPORT_MARKER_PATH) + .await + .map_err(eyre::Report::new) + .wrap_err("acquire vault import lock")?; + + if is_import_complete(db_pool).await? { + tracing::info!("Vault import completed by another replica"); + return Ok(()); + } + + // Strict enumeration: any list or read failure aborts the boot rather + // than importing a subset and recording it as complete. The marker is + // permanent, so a partial import here would be silent credential loss. + let vault_secrets = vault_client + .get_secrets_strict() + .await + .map_err(eyre::Report::from) + .wrap_err("enumerate vault secrets for import")?; + + if vault_secrets.is_empty() { + return Err(eyre::eyre!( + "vault enumeration returned no secrets; refusing to record an import from an \ + empty vault. If this site really has no vault secrets, remove import_from \ + from the [secrets] config; otherwise fix vault and restart" + )); + } + + tracing::info!( + count = vault_secrets.len(), + approach = ?secrets_config.import_approach, + "Importing secrets from vault" + ); + + let result = crate::secrets::import_secrets( + db_pool, + routing, + kms, + &vault_secrets, + secrets_config.import_approach, + ) + .await + .map_err(eyre::Report::new) + .wrap_err("vault secret import")?; + + tracing::info!( + imported = result.imported, + skipped = result.skipped, + "Vault secret import completed" + ); + + crate::secrets::mark_vault_import_complete(db_pool, routing, kms) + .await + .map_err(eyre::Report::new) + .wrap_err("mark vault import complete")?; + tracing::info!("Vault import marked complete"); + + // lock_conn drops here, closing the connection and releasing the + // session advisory lock. + Ok(()) +} + +async fn is_import_complete(db_pool: &PgPool) -> eyre::Result { + crate::secrets::is_vault_import_complete(db_pool) + .await + .map_err(eyre::Report::new) + .wrap_err("check vault import status") +} diff --git a/crates/api-core/src/secrets/import.rs b/crates/api-core/src/secrets/import.rs new file mode 100644 index 0000000000..5974ce0a84 --- /dev/null +++ b/crates/api-core/src/secrets/import.rs @@ -0,0 +1,118 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! One-time import of Vault secrets into the Postgres journal. `run()` +//! drives this at startup: with `[secrets]` configured, the import either +//! completes (recording a permanent marker) before the process serves +//! traffic, or the process does not start. Vault is never part of the +//! credential chain in this mode. + +use carbide_kms_provider::KmsBackend; +use carbide_secrets::credentials::Credentials; +use sqlx::PgPool; +use zeroize::Zeroizing; + +use super::routing::SecretRouting; +use super::{ImportApproach, ImportResult, PgSecretsError, VAULT_IMPORT_MARKER_PATH}; + +/// Import pre-read secrets into Postgres. +/// +/// With `MissingOnly`, a path that already has entries is skipped before +/// any encryption happens; the per-path advisory lock inside +/// `insert_if_missing` keeps a concurrent writer from sneaking an entry in +/// between the check and the insert. With `All`, every secret appends a new +/// journal entry unconditionally. +pub async fn import_secrets( + pool: &PgPool, + routing: &SecretRouting, + kms: &dyn KmsBackend, + secrets: &[(String, Credentials)], + approach: ImportApproach, +) -> Result { + let mut result = ImportResult::default(); + + for (path, credentials) in secrets { + // Cheap existence pre-check so MissingOnly re-imports skip the + // DEK generation and encryption for secrets that already landed. + // The locked check inside insert_if_missing is still the one that + // decides. + if matches!(approach, ImportApproach::MissingOnly) + && db::secrets::exists(pool, path).await? + { + result.skipped += 1; + continue; + } + + let json_bytes = Zeroizing::new(serde_json::to_vec(credentials)?); + let envelope = super::encrypt_envelope(routing, kms, path, &json_bytes).await?; + + match approach { + ImportApproach::MissingOnly => { + let mut txn = pool + .begin() + .await + .map_err(|e| PgSecretsError::Database(db::DatabaseError::acquire(e)))?; + let inserted = + db::secrets::insert_if_missing(&mut txn, &envelope.as_new_entry(path)).await?; + txn.commit().await.map_err(|e| { + PgSecretsError::Database(db::DatabaseError::new("commit import", e)) + })?; + if inserted { + result.imported += 1; + } else { + result.skipped += 1; + } + } + ImportApproach::All => { + let mut conn = pool + .acquire() + .await + .map_err(|e| PgSecretsError::Database(db::DatabaseError::acquire(e)))?; + db::secrets::insert(&mut conn, &envelope.as_new_entry(path)).await?; + result.imported += 1; + } + } + } + + Ok(result) +} + +/// Whether the vault import has already completed (the marker secret +/// exists). +pub async fn is_vault_import_complete(pool: &PgPool) -> Result { + Ok(db::secrets::exists(pool, VAULT_IMPORT_MARKER_PATH).await?) +} + +/// Record vault import completion by writing the marker secret. The marker +/// is an ordinary encrypted journal entry, so it needs no schema of its +/// own. +pub async fn mark_vault_import_complete( + pool: &PgPool, + routing: &SecretRouting, + kms: &dyn KmsBackend, +) -> Result<(), PgSecretsError> { + let envelope = + super::encrypt_envelope(routing, kms, VAULT_IMPORT_MARKER_PATH, b"completed").await?; + + let mut conn = pool + .acquire() + .await + .map_err(|e| PgSecretsError::Database(db::DatabaseError::acquire(e)))?; + + db::secrets::insert(&mut conn, &envelope.as_new_entry(VAULT_IMPORT_MARKER_PATH)).await?; + Ok(()) +} diff --git a/crates/api-core/src/secrets/metrics.rs b/crates/api-core/src/secrets/metrics.rs new file mode 100644 index 0000000000..e0800431ae --- /dev/null +++ b/crates/api-core/src/secrets/metrics.rs @@ -0,0 +1,109 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use std::time::Instant; + +use opentelemetry::KeyValue; +use opentelemetry::metrics::{Counter, Histogram, Meter}; + +/// Request, outcome, and duration instruments for the Postgres secrets +/// backend, under `carbide-api.secrets.*` with an `operation` label +/// (get/set/create/delete). +#[derive(Clone)] +pub struct SecretsMetrics { + requests: Counter, + successes: Counter, + failures: Counter, + duration: Histogram, +} + +impl SecretsMetrics { + /// Create the instruments from an OpenTelemetry meter. + pub fn new(meter: &Meter) -> Self { + Self { + requests: meter + .u64_counter("carbide-api.secrets.requests") + .with_description("Total number of Postgres secrets operations attempted.") + .build(), + successes: meter + .u64_counter("carbide-api.secrets.requests.succeeded") + .with_description("Number of Postgres secrets operations that succeeded.") + .build(), + failures: meter + .u64_counter("carbide-api.secrets.requests.failed") + .with_description("Number of Postgres secrets operations that failed.") + .build(), + duration: meter + .u64_histogram("carbide-api.secrets.request_duration") + .with_description("Duration of Postgres secrets operations, in milliseconds.") + .with_unit("ms") + .build(), + } + } +} + +/// Times one secrets operation and records its outcome exactly once: call +/// [`OperationTimer::succeed`] on the success path, and any other way out +/// of scope -- early `?` returns included -- records a failure on drop. No +/// instruments are touched when metrics are disabled. +pub struct OperationTimer { + metrics: Option, + operation: &'static str, + started: Instant, + completed: bool, +} + +impl OperationTimer { + /// Start timing an operation, counting the request immediately. Pass + /// None to make every recording call a no-op. + pub fn start(metrics: Option, operation: &'static str) -> Self { + if let Some(m) = &metrics { + m.requests.add(1, &[KeyValue::new("operation", operation)]); + } + Self { + metrics, + operation, + started: Instant::now(), + completed: false, + } + } + + /// Record a successful operation and its duration. + pub fn succeed(mut self) { + self.completed = true; + if let Some(m) = &self.metrics { + let elapsed = self.started.elapsed().as_millis() as u64; + let attrs = [KeyValue::new("operation", self.operation)]; + m.successes.add(1, &attrs); + m.duration.record(elapsed, &attrs); + } + } +} + +impl Drop for OperationTimer { + fn drop(&mut self) { + if self.completed { + return; + } + if let Some(m) = &self.metrics { + let elapsed = self.started.elapsed().as_millis() as u64; + let attrs = [KeyValue::new("operation", self.operation)]; + m.failures.add(1, &attrs); + m.duration.record(elapsed, &attrs); + } + } +} diff --git a/crates/api-core/src/secrets/mod.rs b/crates/api-core/src/secrets/mod.rs new file mode 100644 index 0000000000..975ea5e160 --- /dev/null +++ b/crates/api-core/src/secrets/mod.rs @@ -0,0 +1,454 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Credential storage in Postgres, replacing Vault KV. +//! +//! Values are envelope-encrypted: every write generates a new data +//! encryption key (DEK), encrypts the credential with it, and asks the KMS +//! backend to wrap the DEK under a key encryption key (KEK). The row +//! records which KEK wrapped it, so reads never consult routing -- routing +//! ([`SecretRouting`]) only decides which KEK new writes use, and the +//! ciphertext is bound to its path so a row copied onto another path will +//! not decrypt. +//! +//! Reads keep two behaviors the rest of the system learned from the Vault +//! reader: the newest journal entry wins (ordered by the database-assigned +//! `seq`), and an entry whose password is empty reads as no credential at +//! all -- several delete flows "delete" by writing an empty-password +//! tombstone. + +use std::sync::Arc; + +use async_trait::async_trait; +use carbide_kms_provider::{EncryptedDek, KmsBackend}; +use carbide_secrets::SecretsError; +use carbide_secrets::credentials::{ + CredentialKey, CredentialManager, CredentialReader, CredentialWriter, Credentials, +}; +use db::secrets::NewSecretEntry; +use model::secrets::SecretRow; +use serde::Deserialize; +use sqlx::PgPool; +use zeroize::Zeroizing; + +pub mod import; +pub mod metrics; +pub mod re_wrap; +pub mod routing; +#[cfg(test)] +mod tests; + +pub use import::{import_secrets, is_vault_import_complete, mark_vault_import_complete}; +pub use metrics::{OperationTimer, SecretsMetrics}; +pub use re_wrap::{ReWrapStaleResult, re_wrap_stale}; +pub use routing::SecretRouting; + +/// The KMS and routing handles that secrets admin operations (re-wrap) +/// need. None on the `Api` when the `[secrets]` config section is absent. +pub struct SecretsContext { + pub routing: SecretRouting, + pub kms: Arc, +} + +/// The secret path that records vault import completion. It starts with a +/// slash on purpose: real credential paths never do, so no `CredentialKey` +/// can collide with it, and the kek-scoped journal queries exclude it. +pub(crate) const VAULT_IMPORT_MARKER_PATH: &str = "/_vault_import"; + +/// How to treat secrets that already exist in Postgres during an import. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ImportApproach { + /// Import only secrets whose path has no entries yet. + #[default] + MissingOnly, + /// Import everything, appending a new journal entry per secret even + /// when the path already has entries. + All, +} + +/// What an import did. +#[derive(Debug, Default)] +pub struct ImportResult { + /// Secrets written to Postgres. + pub imported: u64, + /// Secrets left alone because their path already had entries + /// (`MissingOnly` only). + pub skipped: u64, +} + +/// Errors from the Postgres secrets backend. +#[derive(Debug, thiserror::Error)] +pub enum PgSecretsError { + #[error("database error: {0}")] + Database(#[from] db::DatabaseError), + + #[error("routing configuration error: {0}")] + RoutingConfig(String), + + #[error("credential already exists for path: {0}")] + AlreadyExists(String), + + #[error("a re-wrap is already running")] + ReWrapInProgress, + + #[error("serialization error: {0}")] + Serialization(#[from] serde_json::Error), + + #[error("KMS error: {0}")] + Kms(#[from] carbide_kms_provider::KmsError), +} + +impl From for SecretsError { + fn from(err: PgSecretsError) -> Self { + SecretsError::GenericError(eyre::Report::new(err)) + } +} + +/// A decrypted journal entry, returned by the history and lookup methods. +pub struct SecretEntry { + /// Identifies this journal entry. + pub secret_id: carbide_uuid::secret::SecretId, + /// The journal order -- higher means written later. + pub seq: i64, + /// The credential path. + pub path: String, + /// The decrypted credential value. + pub credentials: Credentials, + /// The KEK that wrapped this entry's DEK. + pub kek_id: String, + /// When this entry was written. + pub created_at: chrono::DateTime, +} + +/// The `CredentialManager` backed by the Postgres secrets journal. Reads +/// return the newest entry for a path; writes append; delete removes the +/// path's whole history -- the same semantics Vault KV gave the rest of the +/// system. +#[derive(Clone)] +pub struct PostgresCredentialManager { + pool: PgPool, + routing: SecretRouting, + kms: Arc, + metrics: Option, +} + +impl PostgresCredentialManager { + /// Create a manager from a pool, write routing, and KMS backend. + pub fn new(pool: PgPool, routing: SecretRouting, kms: Arc) -> Self { + Self { + pool, + routing, + kms, + metrics: None, + } + } + + /// Attach metrics instruments. + pub fn with_metrics(mut self, metrics: SecretsMetrics) -> Self { + self.metrics = Some(metrics); + self + } + + fn timer(&self, operation: &'static str) -> OperationTimer { + OperationTimer::start(self.metrics.clone(), operation) + } + + async fn conn(&self) -> Result, PgSecretsError> { + self.pool + .acquire() + .await + .map_err(|e| PgSecretsError::Database(db::DatabaseError::acquire(e))) + } + + // -- Journal access, used by credential rotation -- + // + // Nothing in this crate calls these yet: the rotation manager reads + // history to inspect previous values and deletes a failed attempt's + // entry by id, which makes the previous entry current again. + + /// Return every journal entry for a credential, newest first. + pub async fn get_history( + &self, + key: &CredentialKey, + ) -> Result, PgSecretsError> { + let path = key.to_key_str(); + let rows = db::secrets::get_history(&self.pool, &path).await?; + self.decrypt_rows(rows).await + } + + /// Return one journal entry by id. + pub async fn get_by_id( + &self, + secret_id: carbide_uuid::secret::SecretId, + ) -> Result, PgSecretsError> { + let Some(row) = db::secrets::get_by_id(&self.pool, secret_id).await? else { + return Ok(None); + }; + Ok(Some(self.decrypt_row(row).await?)) + } + + /// Return every journal entry wrapped by the given KEK. + pub async fn get_all_for_kek_id( + &self, + kek_id: &str, + ) -> Result, PgSecretsError> { + let rows = db::secrets::get_all_for_kek_id(&self.pool, kek_id).await?; + self.decrypt_rows(rows).await + } + + /// Return the credentials whose newest journal entry is wrapped by the + /// given KEK. + pub async fn get_latest_with_kek_id( + &self, + kek_id: &str, + ) -> Result, PgSecretsError> { + let rows = db::secrets::get_latest_with_kek_id(&self.pool, kek_id).await?; + self.decrypt_rows(rows).await + } + + /// Remove one journal entry by id. Deleting the newest entry makes the + /// previous one current again. + pub async fn delete_by_id( + &self, + secret_id: carbide_uuid::secret::SecretId, + ) -> Result { + let mut conn = self.conn().await?; + Ok(db::secrets::delete_by_id(&mut conn, secret_id).await?) + } + + // -- Envelope encryption -- + + /// Decrypt one row: unwrap its DEK through the KMS, then decrypt the + /// value with the row's path as associated data. + async fn decrypt_row(&self, row: SecretRow) -> Result { + let dek = self + .kms + .decrypt_dek( + &row.kek_id, + &EncryptedDek { + ciphertext: row.encrypted_dek, + nonce: row.dek_nonce, + }, + ) + .await?; + let plaintext = Zeroizing::new(carbide_kms_provider::crypto::decrypt( + &dek, + &row.nonce, + &row.encrypted_value, + row.path.as_bytes(), + )?); + + let credentials: Credentials = serde_json::from_slice(&plaintext)?; + Ok(SecretEntry { + secret_id: row.secret_id, + seq: row.seq, + path: row.path, + credentials, + kek_id: row.kek_id, + created_at: row.created_at, + }) + } + + async fn decrypt_rows(&self, rows: Vec) -> Result, PgSecretsError> { + let mut entries = Vec::with_capacity(rows.len()); + for row in rows { + entries.push(self.decrypt_row(row).await?); + } + Ok(entries) + } + + async fn encrypt_envelope( + &self, + path: &str, + data: &[u8], + ) -> Result { + encrypt_envelope(&self.routing, self.kms.as_ref(), path, data).await + } +} + +/// Encrypt a credential value for `path`: route to the active KEK, generate +/// and wrap a new DEK, and encrypt the value with the path as associated +/// data. Every write -- manager or import -- goes through here, so the +/// path binding cannot diverge between them. +pub(crate) async fn encrypt_envelope( + routing: &SecretRouting, + kms: &dyn KmsBackend, + path: &str, + data: &[u8], +) -> Result { + let kek_id = routing.active_kek_for_path(path)?; + let (dek, wrapped_dek) = kms.generate_and_wrap_dek(kek_id).await?; + let (encrypted_value, nonce) = + carbide_kms_provider::crypto::encrypt(&dek, data, path.as_bytes())?; + Ok(EncryptedEnvelope { + encrypted_value, + nonce, + encrypted_dek: wrapped_dek.ciphertext, + dek_nonce: wrapped_dek.nonce, + kek_id: kek_id.to_string(), + }) +} + +/// The columns produced by one envelope encryption, ready to insert. +pub(crate) struct EncryptedEnvelope { + encrypted_value: Vec, + nonce: Vec, + encrypted_dek: Vec, + dek_nonce: Vec, + kek_id: String, +} + +impl EncryptedEnvelope { + pub(crate) fn as_new_entry<'a>(&'a self, path: &'a str) -> NewSecretEntry<'a> { + NewSecretEntry { + path, + encrypted_value: &self.encrypted_value, + nonce: &self.nonce, + kek_id: &self.kek_id, + encrypted_dek: &self.encrypted_dek, + dek_nonce: &self.dek_nonce, + } + } +} + +#[async_trait] +impl CredentialReader for PostgresCredentialManager { + async fn get_credentials( + &self, + key: &CredentialKey, + ) -> Result, SecretsError> { + let timer = self.timer("get"); + let path = key.to_key_str(); + + let row = db::secrets::get_latest(&self.pool, &path) + .await + .map_err(PgSecretsError::from)?; + let Some(row) = row else { + timer.succeed(); + return Ok(None); + }; + + tracing::debug!( + path = %row.path, + secret_id = %row.secret_id, + seq = row.seq, + kek_id = %row.kek_id, + created_at = %row.created_at, + "read secret from postgres" + ); + + let entry = self.decrypt_row(row).await?; + + timer.succeed(); + // An empty password reads as no credential at all, exactly like the + // Vault reader: the UFM and site-wide BMC delete flows "delete" by + // writing an empty-password tombstone, and their consumers depend + // on getting None back. + match entry.credentials { + Credentials::UsernamePassword { ref password, .. } if password.is_empty() => Ok(None), + credentials => Ok(Some(credentials)), + } + } +} + +#[async_trait] +impl CredentialWriter for PostgresCredentialManager { + async fn set_credentials( + &self, + key: &CredentialKey, + credentials: &Credentials, + ) -> Result<(), SecretsError> { + let timer = self.timer("set"); + let path = key.to_key_str(); + + let json_bytes = + Zeroizing::new(serde_json::to_vec(credentials).map_err(PgSecretsError::from)?); + let envelope = self.encrypt_envelope(&path, &json_bytes).await?; + + let mut conn = self.conn().await?; + db::secrets::insert(&mut conn, &envelope.as_new_entry(&path)) + .await + .map_err(PgSecretsError::from)?; + + timer.succeed(); + Ok(()) + } + + async fn create_credentials( + &self, + key: &CredentialKey, + credentials: &Credentials, + ) -> Result<(), SecretsError> { + let timer = self.timer("create"); + let path = key.to_key_str(); + + // Encrypt before the transaction opens: the KMS call can be a + // network round-trip (Transit), and nothing network-bound belongs + // inside a transaction that holds an advisory lock. The price is + // one wasted envelope when the credential turns out to exist. + let json_bytes = + Zeroizing::new(serde_json::to_vec(credentials).map_err(PgSecretsError::from)?); + let envelope = self.encrypt_envelope(&path, &json_bytes).await?; + + // Create-only means check-then-insert, and those are two + // statements: hold the path's advisory lock for the transaction so + // a concurrent create cannot slip between them. Vault gave us this + // through its compare-and-set; Postgres needs the lock because the + // journal has no unique index to enforce it. + let mut txn = self + .pool + .begin() + .await + .map_err(|e| PgSecretsError::Database(db::DatabaseError::acquire(e)))?; + db::secrets::lock_path(&mut txn, &path) + .await + .map_err(PgSecretsError::from)?; + + if db::secrets::exists(&mut *txn, &path) + .await + .map_err(PgSecretsError::from)? + { + return Err(PgSecretsError::AlreadyExists(path.to_string()).into()); + } + + db::secrets::insert(&mut txn, &envelope.as_new_entry(&path)) + .await + .map_err(PgSecretsError::from)?; + txn.commit() + .await + .map_err(|e| PgSecretsError::Database(db::DatabaseError::new("commit create", e)))?; + + timer.succeed(); + Ok(()) + } + + async fn delete_credentials(&self, key: &CredentialKey) -> Result<(), SecretsError> { + let timer = self.timer("delete"); + let path = key.to_key_str(); + + let mut conn = self.conn().await?; + db::secrets::delete_all(&mut conn, &path) + .await + .map_err(PgSecretsError::from)?; + + timer.succeed(); + Ok(()) + } +} + +impl CredentialManager for PostgresCredentialManager {} diff --git a/crates/api-core/src/secrets/re_wrap.rs b/crates/api-core/src/secrets/re_wrap.rs new file mode 100644 index 0000000000..c138c8c5e9 --- /dev/null +++ b/crates/api-core/src/secrets/re_wrap.rs @@ -0,0 +1,160 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use carbide_kms_provider::{EncryptedDek, KmsBackend}; +use carbide_uuid::secret::SecretId; +use sqlx::PgPool; + +use super::PgSecretsError; +use super::routing::SecretRouting; + +/// The internal path whose advisory lock makes re-wrap single-flight. Like +/// the import marker, it starts with a slash so it can never collide with +/// a credential path -- but unlike the marker, no row is ever written for +/// it; only its lock is used. +const RE_WRAP_LOCK_PATH: &str = "/_re_wrap"; + +/// What a re-wrap pass did, in journal rows. +pub struct ReWrapStaleResult { + /// Rows whose DEK was re-wrapped to the routed KEK. + pub re_wrapped: u64, + /// Rows already wrapped by the routed KEK. + pub already_current: u64, + /// Rows still wrapped by a KEK outside the routing config after the + /// walk. Zero means every unrouted KEK can be retired; nonzero right + /// after a run means concurrent writers landed rows mid-walk -- run + /// re-wrap again once the fleet's config has converged. + pub stale_remaining: u64, +} + +/// A re-wrapped DEK waiting to be written back to its row. +struct PendingReWrap { + secret_id: SecretId, + wrapped: EncryptedDek, + kek_id: String, +} + +/// Re-wrap every journal row whose KEK no longer matches what routing +/// assigns its path -- the operator's one verb after rotating a key in +/// config: make the table agree with the config. +/// +/// Only the DEK-wrapping columns change; the encrypted values are never +/// touched. The table is walked once in journal order, each batch's KMS +/// work happens before its write transaction opens (with Transit, those +/// are network calls that must not run while a transaction is held), and +/// batches commit independently, so an interrupted run keeps its progress. +/// Historical journal entries are re-wrapped too: they must stay +/// decryptable, and re-wrapping them is what lets an old KEK be retired +/// completely. +pub async fn re_wrap_stale( + pool: &PgPool, + kms: &dyn KmsBackend, + routing: &SecretRouting, + batch_size: i64, +) -> Result { + // One re-wrap at a time: a second concurrent run would double every + // KMS round-trip for no benefit. The guard is a session advisory lock + // held on a dedicated connection, not a transaction -- the walk awaits + // Vault/KMS and opens a transaction per batch, and a lock transaction + // held across all of that would trip `txn_held_across_await` and could + // starve the pool. Detaching the connection guarantees the lock + // releases when it drops, including on an early error return. + let mut lock_conn = pool + .acquire() + .await + .map_err(|e| PgSecretsError::Database(db::DatabaseError::acquire(e)))? + .detach(); + if !db::secrets::try_lock_path_session(&mut lock_conn, RE_WRAP_LOCK_PATH).await? { + return Err(PgSecretsError::ReWrapInProgress); + } + + let mut result = ReWrapStaleResult { + re_wrapped: 0, + already_current: 0, + stale_remaining: 0, + }; + + let mut cursor: Option = None; + loop { + let batch = db::secrets::find_batch_after(pool, cursor, batch_size).await?; + let Some(last) = batch.last() else { + break; + }; + cursor = Some(last.seq); + + // Unwrap and re-wrap stale DEKs first, against rows as read -- + // no transaction is open yet. + let mut pending = Vec::new(); + for row in &batch { + let target_kek = routing.active_kek_for_path(&row.path)?; + if row.kek_id == target_kek { + result.already_current += 1; + continue; + } + + let dek = kms + .decrypt_dek( + &row.kek_id, + &EncryptedDek { + ciphertext: row.encrypted_dek.clone(), + nonce: row.dek_nonce.clone(), + }, + ) + .await?; + let wrapped = kms.encrypt_dek(target_kek, &dek).await?; + pending.push(PendingReWrap { + secret_id: row.secret_id, + wrapped, + kek_id: target_kek.to_string(), + }); + } + + if pending.is_empty() { + continue; + } + + // Then one short, write-only transaction per batch. + let mut txn = pool + .begin() + .await + .map_err(|e| PgSecretsError::Database(db::DatabaseError::acquire(e)))?; + for rewrap in &pending { + db::secrets::update_dek_wrap( + &mut txn, + rewrap.secret_id, + &rewrap.wrapped.ciphertext, + &rewrap.wrapped.nonce, + &rewrap.kek_id, + ) + .await?; + } + txn.commit().await.map_err(|e| { + PgSecretsError::Database(db::DatabaseError::new("commit re-wrap batch", e)) + })?; + result.re_wrapped += pending.len() as u64; + } + + // Report what is still wrapped by KEKs outside the routing config -- + // the operator's retire-the-old-key signal. + let routed: Vec = routing + .routes() + .map(|(_, kek_id)| kek_id.to_string()) + .collect(); + result.stale_remaining = db::secrets::count_wrapped_outside(pool, &routed).await? as u64; + + Ok(result) +} diff --git a/crates/api-core/src/secrets/routing.rs b/crates/api-core/src/secrets/routing.rs new file mode 100644 index 0000000000..0d9d9ad3a7 --- /dev/null +++ b/crates/api-core/src/secrets/routing.rs @@ -0,0 +1,296 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use std::collections::HashMap; + +use super::PgSecretsError; + +/// Maps secret path prefixes to the KEK that encrypts new writes under +/// them, longest prefix winning. Routing only decides the key for writes +/// (and the target key for re-wrap) -- reads never consult it, because +/// every stored row records the KEK that wrapped it. +/// +/// Prefixes match whole path segments ("machines/bmc" matches +/// `machines/bmc/...` but not `machines/bmc-archive/...`), and a `"/"` +/// catch-all route is required so that every path -- including ones for +/// credential types that did not exist when the config was written -- has +/// a key to write with. +#[derive(Clone)] +pub struct SecretRouting { + /// `(prefix, kek_id)` sorted longest-prefix-first, with the `"/"` + /// catch-all stored as the empty prefix so it matches everything and + /// sorts last. + routes: Vec<(String, String)>, +} + +/// The route key that matches every path. Secret paths are Vault-style with +/// no leading slash (`machines/bmc/...`), so `"/"` is purely the config +/// spelling for "everything else". +const CATCH_ALL: &str = "/"; + +/// Normalize a config prefix for matching: the catch-all becomes the empty +/// prefix (matches everything, sorts last), and every other prefix gets a +/// trailing slash so it matches whole path segments -- "machines/bmc" must +/// not capture a sibling like `machines/bmc-archive/`. +fn normalize_prefix(prefix: &str) -> String { + if prefix == CATCH_ALL { + String::new() + } else if prefix.ends_with('/') { + prefix.to_string() + } else { + format!("{prefix}/") + } +} + +impl SecretRouting { + /// Build routing from the `[secrets.routing]` config map. Requires a + /// `"/"` catch-all entry, and rejects empty prefixes, empty kek_ids, + /// and prefixes that collide once normalized -- "machines/bmc" and + /// "machines/bmc/" are distinct TOML keys but the same route, and + /// letting both through would pick a winner at random. + pub fn from_config(routing: &HashMap) -> Result { + if !routing.contains_key(CATCH_ALL) { + return Err(PgSecretsError::RoutingConfig(format!( + "routing must include a {CATCH_ALL:?} catch-all entry; without one, \ + writes to unrouted paths have no key and fail at runtime" + ))); + } + + let mut seen: HashMap = HashMap::new(); + for (prefix, kek_id) in routing { + if prefix.is_empty() { + return Err(PgSecretsError::RoutingConfig( + "empty routing prefix; use \"/\" for the catch-all".to_string(), + )); + } + if kek_id.is_empty() { + return Err(PgSecretsError::RoutingConfig(format!( + "empty kek_id for prefix {prefix:?}" + ))); + } + // Credential paths never start with a slash ("machines/..."), + // so a leading-slash prefix other than the catch-all could + // never match -- its writes would silently fall through to the + // catch-all KEK. Reject it rather than encrypt under the wrong + // key. + if prefix != CATCH_ALL && prefix.starts_with('/') { + return Err(PgSecretsError::RoutingConfig(format!( + "routing prefix {prefix:?} starts with '/' but credential paths do not; \ + use \"/\" only for the catch-all" + ))); + } + let normalized = normalize_prefix(prefix); + if let Some(other) = seen.insert(normalized, prefix) { + return Err(PgSecretsError::RoutingConfig(format!( + "routing prefixes {other:?} and {prefix:?} are the same route" + ))); + } + } + + Ok(Self::new( + routing + .iter() + .map(|(prefix, kek_id)| (prefix.clone(), kek_id.clone())) + .collect(), + )) + } + + /// Build routing from pre-built `(prefix, kek_id)` entries. Unlike + /// [`SecretRouting::from_config`] this does not require a catch-all or + /// reject collisions, which lets tests construct partial routing on + /// purpose. + pub fn new(routes: Vec<(String, String)>) -> Self { + let mut routes: Vec<(String, String)> = routes + .into_iter() + .map(|(prefix, kek_id)| (normalize_prefix(&prefix), kek_id)) + .collect(); + // Longest prefix first; the prefix itself breaks length ties so the + // order never depends on HashMap iteration. + routes.sort_by(|a, b| b.0.len().cmp(&a.0.len()).then_with(|| a.0.cmp(&b.0))); + Self { routes } + } + + /// Return the kek_id that encrypts a new write at `path`, using the + /// longest matching prefix. + pub fn active_kek_for_path(&self, path: &str) -> Result<&str, PgSecretsError> { + self.routes + .iter() + .find(|(prefix, _)| path.starts_with(prefix.as_str())) + .map(|(_, kek_id)| kek_id.as_str()) + .ok_or_else(|| { + PgSecretsError::RoutingConfig(format!("no routing prefix matches path {path:?}")) + }) + } + + /// Iterate the configured `(prefix, kek_id)` routes. Startup validation + /// uses this to confirm every routed KEK actually exists in the KMS. + pub fn routes(&self) -> impl Iterator { + self.routes + .iter() + .map(|(prefix, kek_id)| (prefix.as_str(), kek_id.as_str())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // Verifies that the most specific prefix wins (longest-prefix-match). + #[test] + fn longest_prefix_match() { + let routing = SecretRouting::new(vec![ + ("/".to_string(), "default-key".to_string()), + ("machines/bmc".to_string(), "bmc-key".to_string()), + ]); + + assert_eq!( + routing + .active_kek_for_path("machines/bmc/aa:bb/root") + .expect("bmc"), + "bmc-key" + ); + assert_eq!( + routing + .active_kek_for_path("ufm/fabric/auth") + .expect("default"), + "default-key" + ); + } + + // Verifies that the catch-all matches paths no specific prefix does, + // including the internal import-marker path. + #[test] + fn catch_all_matches_everything() { + let routing = SecretRouting::new(vec![ + ("/".to_string(), "default-key".to_string()), + ("a".to_string(), "a-key".to_string()), + ]); + + assert_eq!(routing.active_kek_for_path("a/bc").expect("a"), "a-key"); + assert_eq!( + routing.active_kek_for_path("xyz").expect("rest"), + "default-key" + ); + assert_eq!( + routing + .active_kek_for_path("/_vault_import") + .expect("marker"), + "default-key" + ); + } + + // Verifies that prefixes match whole path segments: a prefix must not + // capture a sibling namespace that merely shares its leading + // characters. + #[test] + fn prefix_does_not_match_mid_segment() { + let routing = SecretRouting::new(vec![ + ("/".to_string(), "default-key".to_string()), + ("machines/bmc".to_string(), "bmc-key".to_string()), + ]); + + assert_eq!( + routing + .active_kek_for_path("machines/bmc/aa:bb/root") + .expect("bmc"), + "bmc-key" + ); + assert_eq!( + routing + .active_kek_for_path("machines/bmc-archive/aa:bb/root") + .expect("sibling"), + "default-key" + ); + } + + // Verifies that from_config rejects a config without a catch-all -- + // without one, some credential writes would have no key. + #[test] + fn from_config_requires_catch_all() { + let mut routing = HashMap::new(); + routing.insert("machines/bmc".to_string(), "bmc-key".to_string()); + assert!(SecretRouting::from_config(&routing).is_err()); + } + + // Verifies that from_config rejects an empty kek_id. + #[test] + fn from_config_empty_kek_id_errors() { + let mut routing = HashMap::new(); + routing.insert("/".to_string(), String::new()); + assert!(SecretRouting::from_config(&routing).is_err()); + } + + // Verifies that two spellings of the same route are rejected instead + // of one silently winning. + #[test] + fn from_config_rejects_colliding_prefixes() { + let mut routing = HashMap::new(); + routing.insert("/".to_string(), "key1".to_string()); + routing.insert("machines/bmc".to_string(), "key2".to_string()); + routing.insert("machines/bmc/".to_string(), "key3".to_string()); + assert!(SecretRouting::from_config(&routing).is_err()); + } + + // Verifies that an empty prefix is rejected -- the catch-all is + // spelled "/". + #[test] + fn from_config_rejects_empty_prefix() { + let mut routing = HashMap::new(); + routing.insert("/".to_string(), "key1".to_string()); + routing.insert(String::new(), "key2".to_string()); + assert!(SecretRouting::from_config(&routing).is_err()); + } + + // Verifies that a non-catch-all prefix starting with "/" is rejected: + // credential paths have no leading slash, so it could never match and + // its writes would silently land under the catch-all KEK. + #[test] + fn from_config_rejects_leading_slash_prefix() { + let mut routing = HashMap::new(); + routing.insert("/".to_string(), "key1".to_string()); + routing.insert("/machines/bmc".to_string(), "key2".to_string()); + assert!(SecretRouting::from_config(&routing).is_err()); + } + + // Verifies that from_config parses a valid config. + #[test] + fn from_config_valid() { + let mut routing = HashMap::new(); + routing.insert("/".to_string(), "key1".to_string()); + routing.insert("machines/bmc".to_string(), "key2".to_string()); + + let r = SecretRouting::from_config(&routing).expect("from_config"); + assert_eq!( + r.active_kek_for_path("machines/bmc/x").expect("bmc"), + "key2" + ); + assert_eq!(r.active_kek_for_path("other").expect("default"), "key1"); + } + + // Verifies that routes() reports the normalized entries for startup + // validation. + #[test] + fn routes_iterates_all_entries() { + let mut routing = HashMap::new(); + routing.insert("/".to_string(), "key1".to_string()); + routing.insert("machines/bmc".to_string(), "key2".to_string()); + + let r = SecretRouting::from_config(&routing).expect("from_config"); + let keks: Vec<&str> = r.routes().map(|(_, kek)| kek).collect(); + assert_eq!(keks, vec!["key2", "key1"]); + } +} diff --git a/crates/api-core/src/secrets/tests.rs b/crates/api-core/src/secrets/tests.rs new file mode 100644 index 0000000000..b071ff0484 --- /dev/null +++ b/crates/api-core/src/secrets/tests.rs @@ -0,0 +1,381 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Database-backed tests for the Postgres credential manager and re-wrap. +//! These build the manager directly on the test pool with local key +//! material -- no API fixture needed. + +use std::collections::HashMap; +use std::sync::Arc; + +use carbide_kms_provider::{IntegratedKmsProvider, KmsBackend}; +use carbide_secrets::credentials::{ + CredentialKey, CredentialReader, CredentialWriter, Credentials, +}; + +use super::PostgresCredentialManager; +use super::re_wrap::re_wrap_stale; +use super::routing::SecretRouting; + +fn test_key(seed: u8) -> [u8; 32] { + let mut key = [0u8; 32]; + for (i, byte) in key.iter_mut().enumerate() { + *byte = seed.wrapping_add(i as u8); + } + key +} + +/// A KMS with one key per (kek_id, seed) pair. +fn kms_with_keys(keys: &[(&str, u8)]) -> Arc { + let map: HashMap = keys + .iter() + .map(|(kek_id, seed)| (kek_id.to_string(), test_key(*seed))) + .collect(); + Arc::new(IntegratedKmsProvider::new(map)) +} + +fn catch_all_routing(kek_id: &str) -> SecretRouting { + SecretRouting::new(vec![("/".to_string(), kek_id.to_string())]) +} + +fn manager( + pool: &sqlx::PgPool, + routing: SecretRouting, + kms: Arc, +) -> PostgresCredentialManager { + PostgresCredentialManager::new(pool.clone(), routing, kms) +} + +fn ufm_key(fabric: &str) -> CredentialKey { + CredentialKey::UfmAuth { + fabric: fabric.to_string(), + } +} + +fn cred(user: &str, pass: &str) -> Credentials { + Credentials::UsernamePassword { + username: user.to_string(), + password: pass.to_string(), + } +} + +// Verifies the journal behavior behind set/get: every set appends, and get +// returns the newest entry. +#[crate::sqlx_test] +async fn set_get_round_trip_and_journal_latest_wins(pool: sqlx::PgPool) { + let mgr = manager(&pool, catch_all_routing("k1"), kms_with_keys(&[("k1", 1)])); + let key = ufm_key("fab1"); + + mgr.set_credentials(&key, &cred("admin", "first")) + .await + .expect("first set"); + mgr.set_credentials(&key, &cred("admin", "second")) + .await + .expect("second set"); + + let current = mgr.get_credentials(&key).await.expect("get"); + assert_eq!(current, Some(cred("admin", "second"))); + + let history = mgr.get_history(&key).await.expect("history"); + assert_eq!(history.len(), 2); + assert_eq!(history[0].credentials, cred("admin", "second")); + assert_eq!(history[1].credentials, cred("admin", "first")); +} + +// Verifies create-only semantics: the second create fails and leaves the +// first value in place. +#[crate::sqlx_test] +async fn create_fails_when_credential_exists(pool: sqlx::PgPool) { + let mgr = manager(&pool, catch_all_routing("k1"), kms_with_keys(&[("k1", 1)])); + let key = ufm_key("fab1"); + + mgr.create_credentials(&key, &cred("admin", "original")) + .await + .expect("first create"); + + let second = mgr + .create_credentials(&key, &cred("admin", "usurper")) + .await; + let err = second.expect_err("second create must fail"); + assert!( + err.to_string().contains("already exists"), + "unexpected error: {err}" + ); + + let current = mgr.get_credentials(&key).await.expect("get"); + assert_eq!(current, Some(cred("admin", "original"))); +} + +// Verifies that delete removes the whole journal, not just the newest +// entry -- the same semantics Vault's delete gave callers. +#[crate::sqlx_test] +async fn delete_removes_all_journal_entries(pool: sqlx::PgPool) { + let mgr = manager(&pool, catch_all_routing("k1"), kms_with_keys(&[("k1", 1)])); + let key = ufm_key("fab1"); + + mgr.set_credentials(&key, &cred("admin", "first")) + .await + .expect("first set"); + mgr.set_credentials(&key, &cred("admin", "second")) + .await + .expect("second set"); + mgr.delete_credentials(&key).await.expect("delete"); + + assert_eq!(mgr.get_credentials(&key).await.expect("get"), None); + assert!(mgr.get_history(&key).await.expect("history").is_empty()); +} + +// Verifies that the journal order is true write order even when two +// entries record the identical created_at (Postgres fixes now() per +// transaction): the second insert wins, by seq, not by chance. +#[crate::sqlx_test] +async fn second_write_wins_on_created_at_ties(pool: sqlx::PgPool) { + let kms = kms_with_keys(&[("k1", 1)]); + let routing = catch_all_routing("k1"); + let path = "tie/test/path"; + + let mut txn = pool.begin().await.expect("begin"); + let first = super::encrypt_envelope(&routing, kms.as_ref(), path, b"\"first\"") + .await + .expect("encrypt first"); + db::secrets::insert(&mut txn, &first.as_new_entry(path)) + .await + .expect("insert first"); + let second = super::encrypt_envelope(&routing, kms.as_ref(), path, b"\"second\"") + .await + .expect("encrypt second"); + db::secrets::insert(&mut txn, &second.as_new_entry(path)) + .await + .expect("insert second"); + txn.commit().await.expect("commit"); + + let newest = db::secrets::get_latest(&pool, path) + .await + .expect("get_latest") + .expect("row"); + assert_eq!( + newest.created_at, + db::secrets::get_history(&pool, path) + .await + .expect("history")[1] + .created_at, + "both entries record the transaction's shared now()" + ); + assert_eq!( + newest.encrypted_value, + second.as_new_entry(path).encrypted_value, + "the later insert must be the newest entry" + ); +} + +// Verifies the rotation rollback story: deleting the newest journal entry +// makes the previous credential current again. +#[crate::sqlx_test] +async fn delete_by_id_restores_previous_credential(pool: sqlx::PgPool) { + let mgr = manager(&pool, catch_all_routing("k1"), kms_with_keys(&[("k1", 1)])); + let key = ufm_key("fab1"); + + mgr.set_credentials(&key, &cred("admin", "v1")) + .await + .expect("set v1"); + mgr.set_credentials(&key, &cred("admin", "v2")) + .await + .expect("set v2"); + + let newest = &mgr.get_history(&key).await.expect("history")[0]; + assert_eq!(newest.credentials, cred("admin", "v2")); + + let fetched = mgr + .get_by_id(newest.secret_id) + .await + .expect("get_by_id") + .expect("entry"); + assert_eq!(fetched.credentials, cred("admin", "v2")); + + assert!(mgr.delete_by_id(newest.secret_id).await.expect("delete")); + assert_eq!( + mgr.get_credentials(&key).await.expect("get"), + Some(cred("admin", "v1")), + "the previous entry is current again" + ); +} + +// Verifies the empty-password tombstone behavior the Vault reader +// established: several delete flows "delete" by writing an empty password, +// and reads must answer None for it. +#[crate::sqlx_test] +async fn empty_password_reads_as_none(pool: sqlx::PgPool) { + let mgr = manager(&pool, catch_all_routing("k1"), kms_with_keys(&[("k1", 1)])); + let key = ufm_key("fab1"); + + mgr.set_credentials(&key, &cred("admin", "live")) + .await + .expect("set live"); + mgr.set_credentials(&key, &cred("admin", "")) + .await + .expect("set tombstone"); + + assert_eq!( + mgr.get_credentials(&key).await.expect("get"), + None, + "an empty-password tombstone must read as no credential" + ); + assert_eq!( + mgr.get_history(&key).await.expect("history").len(), + 2, + "the journal keeps the tombstone entry itself" + ); +} + +// Verifies the associated-data binding end to end: a ciphertext copied +// onto another path fails to decrypt instead of serving the wrong +// credential. +#[crate::sqlx_test] +async fn ciphertext_copied_to_another_path_does_not_decrypt(pool: sqlx::PgPool) { + let mgr = manager(&pool, catch_all_routing("k1"), kms_with_keys(&[("k1", 1)])); + let key_a = ufm_key("fab-a"); + let key_b = ufm_key("fab-b"); + + mgr.set_credentials(&key_a, &cred("admin", "a-secret")) + .await + .expect("set"); + + // Copy fab-a's encrypted columns onto fab-b's path, the way an + // attacker with table access (but no keys) would. + sqlx::query( + "INSERT INTO secrets + (secret_id, path, encrypted_value, nonce, kek_id, encrypted_dek, dek_nonce) + SELECT gen_random_uuid(), $2, encrypted_value, nonce, kek_id, encrypted_dek, dek_nonce + FROM secrets WHERE path = $1", + ) + .bind(key_a.to_key_str().as_ref()) + .bind(key_b.to_key_str().as_ref()) + .execute(&pool) + .await + .expect("transplant row"); + + let stolen = mgr.get_credentials(&key_b).await; + assert!( + stolen.is_err(), + "a transplanted ciphertext must fail decryption, got: {stolen:?}" + ); +} + +// Verifies that re-wrap moves every stale row to the KEK routing assigns, +// the rows still decrypt afterwards, and a second run finds nothing to do. +#[crate::sqlx_test] +async fn re_wrap_stale_moves_rows_and_is_idempotent(pool: sqlx::PgPool) { + let kms = kms_with_keys(&[("old-key", 1), ("new-key", 2)]); + + // Write under old-key: two credentials, one with two journal entries. + let mgr_old = manager(&pool, catch_all_routing("old-key"), kms.clone()); + mgr_old + .set_credentials(&ufm_key("fab1"), &cred("admin", "one")) + .await + .expect("set fab1"); + mgr_old + .set_credentials(&ufm_key("fab1"), &cred("admin", "two")) + .await + .expect("set fab1 again"); + mgr_old + .set_credentials(&ufm_key("fab2"), &cred("admin", "three")) + .await + .expect("set fab2"); + + // Rotate: routing now assigns new-key to everything. + let routing = catch_all_routing("new-key"); + let result = re_wrap_stale(&pool, kms.as_ref(), &routing, 2) + .await + .expect("re-wrap"); + assert_eq!(result.re_wrapped, 3); + assert_eq!(result.already_current, 0); + assert_eq!( + result.stale_remaining, 0, + "old-key is unrouted and nothing is left on it" + ); + + // Every row decrypts, and historical entries moved too. + let mgr_new = manager(&pool, routing.clone(), kms.clone()); + assert_eq!( + mgr_new + .get_credentials(&ufm_key("fab1")) + .await + .expect("get fab1"), + Some(cred("admin", "two")) + ); + assert_eq!( + mgr_new + .get_history(&ufm_key("fab1")) + .await + .expect("history") + .len(), + 2 + ); + assert!( + mgr_new + .get_all_for_kek_id("old-key") + .await + .expect("old rows") + .is_empty() + ); + + // A second run reports everything current and changes nothing. + let again = re_wrap_stale(&pool, kms.as_ref(), &routing, 2) + .await + .expect("re-wrap again"); + assert_eq!(again.re_wrapped, 0); + assert_eq!(again.already_current, 3); +} + +// Verifies that the re-wrap counters are exact when rows move between two +// KEKs that are both still routed -- the single-pass walk classifies each +// row exactly once. +#[crate::sqlx_test] +async fn re_wrap_stale_counts_each_row_once_across_routed_keks(pool: sqlx::PgPool) { + let kms = kms_with_keys(&[("k1", 1), ("k2", 2)]); + + // Both paths start under k1. + let routing_old = catch_all_routing("k1"); + for path in ["alpha/one", "beta/two"] { + let envelope = super::encrypt_envelope(&routing_old, kms.as_ref(), path, b"{}") + .await + .expect("encrypt"); + let mut conn = pool.acquire().await.expect("acquire"); + db::secrets::insert(&mut conn, &envelope.as_new_entry(path)) + .await + .expect("insert"); + } + + // New routing sends beta/ to k2 while k1 stays routed for the rest. + let routing_new = SecretRouting::new(vec![ + ("/".to_string(), "k1".to_string()), + ("beta".to_string(), "k2".to_string()), + ]); + + let result = re_wrap_stale(&pool, kms.as_ref(), &routing_new, 1) + .await + .expect("re-wrap"); + assert_eq!(result.re_wrapped, 1, "only beta/two moved"); + assert_eq!(result.already_current, 1, "alpha/one was already routed"); + assert_eq!(result.stale_remaining, 0); + + let again = re_wrap_stale(&pool, kms.as_ref(), &routing_new, 1) + .await + .expect("re-wrap again"); + assert_eq!(again.re_wrapped, 0); + assert_eq!(again.already_current, 2); +} diff --git a/crates/api-core/src/setup.rs b/crates/api-core/src/setup.rs index 474a926c2f..6aeb14e93e 100644 --- a/crates/api-core/src/setup.rs +++ b/crates/api-core/src/setup.rs @@ -259,7 +259,9 @@ pub fn create_ipmi_tool( /// Configure and create a postgres connection pool /// /// This connects to the database to verify settings -async fn create_and_connect_postgres_pool(config: &CarbideConfig) -> eyre::Result { +pub(crate) async fn create_and_connect_postgres_pool( + config: &CarbideConfig, +) -> eyre::Result { // We need logs to be enabled at least at `INFO` level. Otherwise // our global logging filter would reject the logs before they get injected // into the `SqlxQueryTracing` layer. @@ -294,6 +296,8 @@ pub async fn start_api( shared_nv_redfish_pool: Arc, credential_manager: Arc, certificate_provider: Arc, + db_pool: PgPool, + secrets_context: Option, admin_ui_routes_builder: Option, cancel_token: CancellationToken, ready_channel: Sender<()>, @@ -304,8 +308,6 @@ pub async fn start_api( dynamic_settings.bmc_proxy.clone(), ); - let db_pool = create_and_connect_postgres_pool(&carbide_config).await?; - let work_lock_manager_handle = work_lock_manager::start( join_set, db_pool.clone(), @@ -605,6 +607,7 @@ pub async fn start_api( metric_emitter: ApiMetricsEmitter::new(&meter), component_manager, bms_client: std::sync::OnceLock::new(), + secrets_context, }); if carbide_config.listen_only { diff --git a/crates/api-core/src/test_support/builder.rs b/crates/api-core/src/test_support/builder.rs index 822b87c5f4..47b755f52f 100644 --- a/crates/api-core/src/test_support/builder.rs +++ b/crates/api-core/src/test_support/builder.rs @@ -62,6 +62,7 @@ pub struct TestApiBuilder { metric_emitter: Option, ib_fabric_manager: Option>, component_manager: Option>, + secrets_context: Option, } impl TestApiBuilder { @@ -84,6 +85,7 @@ impl TestApiBuilder { metric_emitter: None, ib_fabric_manager: None, component_manager: None, + secrets_context: None, } } @@ -150,6 +152,17 @@ impl TestApiBuilder { } } + /// Build a secrets-backed `Api` so handler tests can exercise the + /// Postgres secrets / re-wrap path. Left `None` by default, which keeps + /// the secrets RPCs returning "not configured" as in production without + /// a `[secrets]` section. + pub fn with_secrets_context(self, secrets_context: crate::secrets::SecretsContext) -> Self { + Self { + secrets_context: Some(secrets_context), + ..self + } + } + pub fn with_component_manager( self, component_manager: Arc, @@ -251,6 +264,7 @@ impl TestApiBuilder { component_manager: self.component_manager.map(|cm| (*cm).clone()), bmc_session_manager, bms_client: std::sync::OnceLock::new(), + secrets_context: self.secrets_context, } } } diff --git a/crates/api-core/src/test_support/default_config.rs b/crates/api-core/src/test_support/default_config.rs index 1e3724752f..c2b739be3c 100644 --- a/crates/api-core/src/test_support/default_config.rs +++ b/crates/api-core/src/test_support/default_config.rs @@ -252,6 +252,7 @@ pub fn get() -> CarbideConfig { initial_objects_file: None, config_ctx: None, tracing: TracingConfig::default(), + secrets: None, } } diff --git a/crates/api-db/migrations/20260416000305_create_secrets_table.sql b/crates/api-db/migrations/20260416000305_create_secrets_table.sql new file mode 100644 index 0000000000..75ed82a4bc --- /dev/null +++ b/crates/api-db/migrations/20260416000305_create_secrets_table.sql @@ -0,0 +1,41 @@ +-- Encrypted credential storage, replacing Vault KV. +-- +-- Each row is one envelope-encrypted credential value: the value is +-- encrypted with a per-row data encryption key (DEK), and the DEK is in turn +-- wrapped by a key encryption key (KEK) that lives outside the database -- +-- in carbide's own key config or an external KMS. kek_id records which KEK +-- wrapped this row's DEK so rotation can find and re-wrap rows in place. +-- +-- The table is an append-only journal: every write inserts a new row, and a +-- read returns the newest row for the path. seq is the journal order -- +-- created_at cannot be, because Postgres fixes now() per transaction, so +-- two writes in one transaction record the same timestamp. Older rows are +-- kept so that credential rotation can roll back by deleting the newest +-- entry. +-- +-- Paths beginning with "/" are internal bookkeeping entries (the vault +-- import marker), not credentials; real credential paths never start with +-- a slash. +CREATE TABLE secrets ( + secret_id UUID PRIMARY KEY, + -- UNIQUE because seq is the sole cursor for paginated journal walks + -- (find_batch_after's `seq > $1`), the ORDER BY key, and the + -- newest-per-path selector. IDENTITY alone does not guarantee + -- uniqueness (a sequence reset or OVERRIDING SYSTEM VALUE insert could + -- duplicate it), and a duplicate would make pagination skip or repeat + -- rows. + seq BIGINT GENERATED ALWAYS AS IDENTITY UNIQUE, + path TEXT NOT NULL, + encrypted_value BYTEA NOT NULL, + nonce BYTEA NOT NULL, + kek_id TEXT NOT NULL, + encrypted_dek BYTEA NOT NULL, + dek_nonce BYTEA NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- Serves the hot read path: newest row for a path. +CREATE INDEX idx_secrets_path_latest ON secrets (path, seq DESC); + +-- Serves re-wrap scans: all rows wrapped by a given KEK. +CREATE INDEX idx_secrets_kek_id ON secrets (kek_id); diff --git a/crates/api-db/src/lib.rs b/crates/api-db/src/lib.rs index 5276717bf4..2a19cedb76 100644 --- a/crates/api-db/src/lib.rs +++ b/crates/api-db/src/lib.rs @@ -81,6 +81,7 @@ pub mod redfish_actions; pub mod resource_pool; pub mod retained_boot_interface; pub mod route_servers; +pub mod secrets; pub mod site_exploration_report; pub mod sku; pub mod spx_partition; diff --git a/crates/api-db/src/secrets.rs b/crates/api-db/src/secrets.rs new file mode 100644 index 0000000000..732d73d902 --- /dev/null +++ b/crates/api-db/src/secrets.rs @@ -0,0 +1,314 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Queries for the `secrets` table -- an append-only journal of +//! envelope-encrypted credential values. Every write inserts a new row; a +//! read returns the newest row for the path. `seq DESC` is the journal +//! order everywhere: it is assigned by the database at insert, so it +//! follows true insertion order where `created_at` cannot (Postgres fixes +//! `now()` per transaction). +//! +//! Paths beginning with "/" are internal bookkeeping entries (the vault +//! import marker), not credentials. The kek-scoped queries exclude them so +//! callers that decrypt and parse whole result sets never trip over a +//! non-credential payload. + +use carbide_uuid::secret::SecretId; +use model::secrets::SecretRow; +use sqlx::PgConnection; + +use crate::db_read::DbReader; +use crate::{DatabaseError, DatabaseResult}; + +/// The envelope-encryption columns for one journal entry, exactly as the +/// manager produced them. Grouped so the insert paths cannot mix up five +/// consecutive `&[u8]`/`&str` arguments. +pub struct NewSecretEntry<'a> { + pub path: &'a str, + pub encrypted_value: &'a [u8], + pub nonce: &'a [u8], + pub kek_id: &'a str, + pub encrypted_dek: &'a [u8], + pub dek_nonce: &'a [u8], +} + +/// Return the newest entry for a path. +pub async fn get_latest(txn: impl DbReader<'_>, path: &str) -> DatabaseResult> { + let sql = "SELECT * FROM secrets WHERE path = $1 + ORDER BY seq DESC LIMIT 1"; + sqlx::query_as(sql) + .bind(path) + .fetch_optional(txn) + .await + .map_err(|e| DatabaseError::query(sql, e)) +} + +/// Append a new journal entry. +pub async fn insert(txn: &mut PgConnection, entry: &NewSecretEntry<'_>) -> DatabaseResult<()> { + let secret_id = SecretId::new(); + let sql = "INSERT INTO secrets + (secret_id, path, encrypted_value, nonce, + kek_id, encrypted_dek, dek_nonce) + VALUES ($1, $2, $3, $4, $5, $6, $7)"; + sqlx::query(sql) + .bind(secret_id) + .bind(entry.path) + .bind(entry.encrypted_value) + .bind(entry.nonce) + .bind(entry.kek_id) + .bind(entry.encrypted_dek) + .bind(entry.dek_nonce) + .execute(txn) + .await + .map_err(|e| DatabaseError::query(sql, e))?; + Ok(()) +} + +/// Append a new journal entry only if the path has no entries yet. Returns +/// true when the row was inserted, false when entries already existed. +/// +/// The check and the insert are two statements, so this takes a transaction +/// and serializes concurrent callers on a per-path advisory lock -- without +/// it, two racing creates would both pass the existence check and both +/// insert. The lock releases with the transaction. +pub async fn insert_if_missing( + txn: &mut PgConnection, + entry: &NewSecretEntry<'_>, +) -> DatabaseResult { + lock_path(&mut *txn, entry.path).await?; + if exists(&mut *txn, entry.path).await? { + return Ok(false); + } + insert(txn, entry).await?; + Ok(true) +} + +/// Take the transaction-scoped advisory lock for a path. Callers that need +/// check-then-write semantics on a path (there is no unique index -- the +/// journal allows many rows per path) take this lock first so concurrent +/// writers serialize. +/// +/// The hashed string is namespaced with "secrets:" so this can never +/// collide with other subsystems that take advisory locks on their own +/// hashed strings. +pub async fn lock_path(txn: &mut PgConnection, path: &str) -> DatabaseResult<()> { + let sql = "SELECT pg_advisory_xact_lock(hashtextextended('secrets:' || $1, 0))"; + sqlx::query(sql) + .bind(path) + .execute(txn) + .await + .map_err(|e| DatabaseError::query(sql, e))?; + Ok(()) +} + +/// Take the session-scoped advisory lock for a path on this connection, +/// without waiting. Returns false when another session already holds it. +/// +/// Unlike [`lock_path`], the lock outlives any single transaction and is +/// held for as long as the connection stays open -- so a caller can guard +/// a long operation without keeping a transaction open across its awaits. +/// Release it with [`unlock_path_session`], or by dropping the connection. +pub async fn try_lock_path_session(conn: &mut PgConnection, path: &str) -> DatabaseResult { + let sql = "SELECT pg_try_advisory_lock(hashtextextended('secrets:' || $1, 0))"; + sqlx::query_scalar(sql) + .bind(path) + .fetch_one(conn) + .await + .map_err(|e| DatabaseError::query(sql, e)) +} + +/// Take the session-scoped advisory lock for a path on this connection, +/// waiting until it is free. The same release rules as +/// [`try_lock_path_session`] apply. +pub async fn lock_path_session(conn: &mut PgConnection, path: &str) -> DatabaseResult<()> { + let sql = "SELECT pg_advisory_lock(hashtextextended('secrets:' || $1, 0))"; + sqlx::query(sql) + .bind(path) + .execute(conn) + .await + .map_err(|e| DatabaseError::query(sql, e))?; + Ok(()) +} + +/// Release a session-scoped advisory lock taken by [`lock_path_session`] or +/// [`try_lock_path_session`]. Dropping the connection releases it too, so +/// this is only needed when the connection is kept for further work. +pub async fn unlock_path_session(conn: &mut PgConnection, path: &str) -> DatabaseResult<()> { + let sql = "SELECT pg_advisory_unlock(hashtextextended('secrets:' || $1, 0))"; + sqlx::query(sql) + .bind(path) + .execute(conn) + .await + .map_err(|e| DatabaseError::query(sql, e))?; + Ok(()) +} + +/// Whether any entries exist for the path. +pub async fn exists(txn: impl DbReader<'_>, path: &str) -> DatabaseResult { + let sql = "SELECT EXISTS(SELECT 1 FROM secrets WHERE path = $1)"; + sqlx::query_scalar(sql) + .bind(path) + .fetch_one(txn) + .await + .map_err(|e| DatabaseError::query(sql, e)) +} + +/// Remove every journal entry for a path. +pub async fn delete_all(txn: &mut PgConnection, path: &str) -> DatabaseResult<()> { + let sql = "DELETE FROM secrets WHERE path = $1"; + sqlx::query(sql) + .bind(path) + .execute(txn) + .await + .map_err(|e| DatabaseError::query(sql, e))?; + Ok(()) +} + +/// Remove one journal entry by id. Returns true when a row was deleted. +/// Deleting the newest entry makes the previous entry current again, which +/// is how credential rotation rolls back a failed attempt. +pub async fn delete_by_id(txn: &mut PgConnection, secret_id: SecretId) -> DatabaseResult { + let sql = "DELETE FROM secrets WHERE secret_id = $1"; + let result = sqlx::query(sql) + .bind(secret_id) + .execute(txn) + .await + .map_err(|e| DatabaseError::query(sql, e))?; + Ok(result.rows_affected() > 0) +} + +/// Return every journal entry for a path, newest first. +pub async fn get_history(txn: impl DbReader<'_>, path: &str) -> DatabaseResult> { + let sql = "SELECT * FROM secrets WHERE path = $1 + ORDER BY seq DESC"; + sqlx::query_as(sql) + .bind(path) + .fetch_all(txn) + .await + .map_err(|e| DatabaseError::query(sql, e)) +} + +/// Return one journal entry by id. +pub async fn get_by_id( + txn: impl DbReader<'_>, + secret_id: SecretId, +) -> DatabaseResult> { + let sql = "SELECT * FROM secrets WHERE secret_id = $1"; + sqlx::query_as(sql) + .bind(secret_id) + .fetch_optional(txn) + .await + .map_err(|e| DatabaseError::query(sql, e)) +} + +/// Return every credential journal entry whose DEK is wrapped by the given +/// KEK. Internal bookkeeping paths are excluded: callers decrypt and parse +/// these rows as credentials. +pub async fn get_all_for_kek_id( + txn: impl DbReader<'_>, + kek_id: &str, +) -> DatabaseResult> { + let sql = "SELECT * FROM secrets + WHERE kek_id = $1 AND left(path, 1) <> '/' + ORDER BY seq DESC"; + sqlx::query_as(sql) + .bind(kek_id) + .fetch_all(txn) + .await + .map_err(|e| DatabaseError::query(sql, e)) +} + +/// Return the credentials whose newest journal entry is wrapped by the +/// given KEK, one row per path. Internal bookkeeping paths are excluded: +/// callers decrypt and parse these rows as credentials. +pub async fn get_latest_with_kek_id( + txn: impl DbReader<'_>, + kek_id: &str, +) -> DatabaseResult> { + let sql = "SELECT * FROM ( + SELECT DISTINCT ON (path) * + FROM secrets + WHERE left(path, 1) <> '/' + ORDER BY path, seq DESC + ) latest + WHERE latest.kek_id = $1"; + sqlx::query_as(sql) + .bind(kek_id) + .fetch_all(txn) + .await + .map_err(|e| DatabaseError::query(sql, e)) +} + +/// Return one page of the whole table in journal order. Pass the last +/// row's `seq` back as `after_seq` for the next page, or None to start +/// from the beginning. Re-wrap walks the table with this -- including the +/// internal bookkeeping rows, whose DEKs need re-wrapping like any other. +pub async fn find_batch_after( + txn: impl DbReader<'_>, + after_seq: Option, + limit: i64, +) -> DatabaseResult> { + let sql = "SELECT * FROM secrets + WHERE ($1::bigint IS NULL OR seq > $1) + ORDER BY seq LIMIT $2"; + sqlx::query_as(sql) + .bind(after_seq) + .bind(limit) + .fetch_all(txn) + .await + .map_err(|e| DatabaseError::query(sql, e)) +} + +/// Count the rows whose DEK is wrapped by a KEK outside the given set. +/// After a re-wrap, a zero here is the operator's signal that every KEK +/// absent from the routing config can be retired; nonzero right after a +/// re-wrap means concurrent writers landed rows mid-walk -- run it again. +pub async fn count_wrapped_outside( + txn: impl DbReader<'_>, + kek_ids: &[String], +) -> DatabaseResult { + let sql = "SELECT count(*) FROM secrets WHERE kek_id <> ALL($1)"; + sqlx::query_scalar(sql) + .bind(kek_ids) + .fetch_one(txn) + .await + .map_err(|e| DatabaseError::query(sql, e)) +} + +/// Replace the DEK-wrapping columns on one row, leaving the encrypted value +/// untouched. This is the whole write side of KEK rotation: the data never +/// needs re-encrypting, only its DEK needs re-wrapping under the new KEK. +pub async fn update_dek_wrap( + txn: &mut PgConnection, + secret_id: SecretId, + encrypted_dek: &[u8], + dek_nonce: &[u8], + kek_id: &str, +) -> DatabaseResult<()> { + let sql = "UPDATE secrets SET encrypted_dek = $1, + dek_nonce = $2, kek_id = $3 + WHERE secret_id = $4"; + sqlx::query(sql) + .bind(encrypted_dek) + .bind(dek_nonce) + .bind(kek_id) + .bind(secret_id) + .execute(txn) + .await + .map_err(|e| DatabaseError::query(sql, e))?; + Ok(()) +} diff --git a/crates/api-integration-tests/Cargo.toml b/crates/api-integration-tests/Cargo.toml index 19a3e6f354..a46b27f72e 100644 --- a/crates/api-integration-tests/Cargo.toml +++ b/crates/api-integration-tests/Cargo.toml @@ -28,12 +28,16 @@ authors.workspace = true carbide-machine-a-tron = { path = "../machine-a-tron" } carbide-utils = { path = "../utils", features = ["sqlx"] } carbide-api-test-helper = { path = "../api-test-helper" } +carbide-kms-provider = { path = "../kms-provider" } carbide-secrets = { path = "../secrets" } +carbide-api-core = { path = "../api-core", default-features = false } +carbide-api-db = { path = "../api-db", default-features = false } bmc-mock = { path = "../bmc-mock" } # DO NOT PUT DEPENDENCIES OTHER THAN LOCAL DEPS HERE, THEY SHOULD ALL HAVE 'path =' IN THEM. #these are alphabetized askama_escape = { workspace = true } +base64 = { workspace = true } ctor = { workspace = true } eyre = { workspace = true } futures = { workspace = true } diff --git a/crates/api-integration-tests/tests/secrets_import.rs b/crates/api-integration-tests/tests/secrets_import.rs new file mode 100644 index 0000000000..f09caeb940 --- /dev/null +++ b/crates/api-integration-tests/tests/secrets_import.rs @@ -0,0 +1,337 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Integration tests for the Vault-to-Postgres secrets import flow. +//! These tests start a real Vault dev server and connect to a real Postgres instance. +//! Requires: `vault` binary in PATH, `DATABASE_URL` env var set. + +use std::net::{SocketAddr, TcpListener}; + +use carbide_secrets::credentials::{ + BmcCredentialType, CredentialKey, CredentialReader, CredentialType, CredentialWriter, + Credentials, MqttCredentialType, NicLockdownIkm, +}; +use carbide_secrets::{VaultConfig, create_vault_client}; +use sqlx::PgPool; +use sqlx::postgres::PgConnectOptions; +use std::str::FromStr; + +/// allocate_port picks a free port by binding to port 0. +fn allocate_port() -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind to free port"); + listener.local_addr().expect("local addr") +} + +/// make_test_key creates a deterministic 32-byte key from a seed byte. +fn make_test_key(seed: u8) -> [u8; 32] { + let mut key = [0u8; 32]; + for (i, byte) in key.iter_mut().enumerate() { + *byte = seed.wrapping_add(i as u8); + } + key +} + +/// make_routing_and_kms creates a SecretRouting and IntegratedKmsProvider with a "/" catch-all. +fn make_routing_and_kms( + key: [u8; 32], +) -> ( + carbide_api_core::secrets::SecretRouting, + std::sync::Arc, +) { + let kek_id = "import-test-key".to_string(); + let mut keys = std::collections::HashMap::new(); + keys.insert(kek_id.clone(), key); + let kms: std::sync::Arc = + std::sync::Arc::new(carbide_kms_provider::IntegratedKmsProvider::new(keys)); + let routing = carbide_api_core::secrets::SecretRouting::new(vec![("/".to_string(), kek_id)]); + (routing, kms) +} + +/// generate_test_secrets builds a set of CredentialKey/Credentials pairs covering every variant +/// shape, then scales up to at least `min_count` with synthetic dynamic IDs. +fn generate_test_secrets(min_count: usize) -> Vec<(CredentialKey, Credentials)> { + let cred = |user: &str, pass: &str| Credentials::UsernamePassword { + username: user.to_string(), + password: pass.to_string(), + }; + let mac = |i: u8| mac_address::MacAddress::new([0x00, 0x11, 0x22, 0x33, 0x44, i]); + + // Start with one of every static variant shape. + let mut secrets: Vec<(CredentialKey, Credentials)> = vec![ + ( + CredentialKey::BmcCredentials { + credential_type: BmcCredentialType::SiteWideRoot, + }, + cred("bmc-root", "bmc-pass"), + ), + ( + CredentialKey::DpuRedfish { + credential_type: CredentialType::SiteDefault, + }, + cred("dpu-redfish-site", "pass"), + ), + ( + CredentialKey::DpuRedfish { + credential_type: CredentialType::DpuHardwareDefault, + }, + cred("dpu-redfish-hw", "pass"), + ), + ( + CredentialKey::HostRedfish { + credential_type: CredentialType::SiteDefault, + }, + cred("host-redfish-site", "pass"), + ), + ( + CredentialKey::DpuUefi { + credential_type: CredentialType::SiteDefault, + }, + cred("dpu-uefi-site", "pass"), + ), + ( + CredentialKey::DpuUefi { + credential_type: CredentialType::DpuHardwareDefault, + }, + cred("dpu-uefi-hw", "pass"), + ), + ( + CredentialKey::HostUefi { + credential_type: CredentialType::SiteDefault, + }, + cred("host-uefi-site", "pass"), + ), + ( + CredentialKey::MqttAuth { + credential_type: MqttCredentialType::Dpa, + }, + cred("mqtt-dpa", "pass"), + ), + ( + CredentialKey::MqttAuth { + credential_type: MqttCredentialType::DsxExchangeEventBus, + }, + cred("mqtt-event-bus", "pass"), + ), + ( + CredentialKey::MqttAuth { + credential_type: MqttCredentialType::DsxExchangeConsumer, + }, + cred("mqtt-consumer", "pass"), + ), + ]; + + // Scale up with dynamic variants until we hit min_count. + let mut i = 0u8; + while secrets.len() < min_count { + // Rotate through different dynamic variant types. + let key = match i % 7 { + 0 => CredentialKey::UfmAuth { + fabric: format!("fabric-{i}"), + }, + 1 => CredentialKey::BmcCredentials { + credential_type: BmcCredentialType::BmcRoot { + bmc_mac_address: mac(i), + }, + }, + 2 => CredentialKey::BmcCredentials { + credential_type: BmcCredentialType::BmcForgeAdmin { + bmc_mac_address: mac(i), + }, + }, + 3 => CredentialKey::ExtensionService { + service_id: format!("svc-{i}"), + version: format!("v{i}"), + }, + 4 => CredentialKey::NmxM { + nmxm_id: format!("nmxm-{i}"), + }, + 5 => CredentialKey::NicLockdownIkm { + credential_type: NicLockdownIkm::SiteWide { version: i as u32 }, + }, + _ => CredentialKey::SwitchNvosAdmin { + bmc_mac_address: mac(i), + }, + }; + secrets.push((key, cred(&format!("user-{i}"), &format!("pass-{i}")))); + i = i.wrapping_add(1); + } + + secrets +} + +/// Verifies the full vault-to-postgres import flow: +/// 1. Populate Vault with secrets. +/// 2. Import all secrets into Postgres. +/// 3. Verify all secrets are readable from Postgres. +/// 4. Re-import with MissingOnly and verify it's a noop. +/// 5. Verify the /_vault_import marker exists. +#[tokio::test] +async fn vault_to_postgres_import() -> eyre::Result<()> { + // Skip if DATABASE_URL is not set (no Postgres available). + let db_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => { + eprintln!("DATABASE_URL not set, skipping vault_to_postgres_import test"); + return Ok(()); + } + }; + + // Skip if vault binary is not in PATH. + if std::env::split_paths(&std::env::var_os("PATH").unwrap_or_default()) + .filter_map(|dir| { + let candidate = dir.join("vault"); + candidate.is_file().then_some(candidate) + }) + .next() + .is_none() + { + eprintln!("vault binary not found in PATH, skipping vault_to_postgres_import test"); + return Ok(()); + } + + // --- Set up test database --- + // Derive the test DB's connect options from the parsed admin options + // rather than string-concatenating onto DATABASE_URL -- concatenation + // breaks if the URL carries its own database path or query parameters. + let admin_opts = PgConnectOptions::from_str(&db_url)?; + let test_db_name = format!("secrets_import_test_{}", std::process::id()); + let admin_pool = PgPool::connect_with(admin_opts.clone()).await?; + sqlx::query(sqlx::AssertSqlSafe(format!( + "CREATE DATABASE {test_db_name}" + ))) + .execute(&admin_pool) + .await?; + let test_opts = admin_opts.database(&test_db_name); + + // Everything from connecting onward runs in a fallible block whose + // result is captured, so the database is dropped whether the import + // succeeds, an assertion fails, or migrations error -- no leaked + // temporary databases. + let result = async { + let test_pool = PgPool::connect_with(test_opts).await?; + db::migrations::MIGRATOR.run(&test_pool).await?; + let outcome = exercise_import(&test_pool).await; + test_pool.close().await; + outcome + } + .await; + + sqlx::query(sqlx::AssertSqlSafe(format!("DROP DATABASE {test_db_name}"))) + .execute(&admin_pool) + .await?; + + result +} + +/// The import flow itself: populate Vault, import into Postgres, verify +/// round-trips, re-import as a noop, and check the completion marker. +async fn exercise_import(test_pool: &PgPool) -> eyre::Result<()> { + // --- Start Vault --- + let vault_addr = allocate_port(); + let vault = api_test_helper::vault::start(vault_addr).await?; + let vault_config = VaultConfig { + address: Some(format!("https://{vault_addr}")), + kv_mount_location: Some("secret".to_string()), + pki_mount_location: Some("forgeca".to_string()), + pki_role_name: Some("forge-cluster".to_string()), + token: Some(vault.token.clone()), + vault_cacert: Some(vault.ca_cert.clone()), + ..Default::default() + }; + + let meter = opentelemetry::global::meter("secrets-import-test"); + let vault_client = create_vault_client(&vault_config, meter)?; + + // --- Populate Vault with secrets --- + let secrets = generate_test_secrets(100); + for (key, cred) in &secrets { + vault_client.set_credentials(key, cred).await?; + } + eprintln!("Populated {} secrets in Vault", secrets.len()); + + // --- List all secrets from Vault --- + let vault_secrets = vault_client.get_secrets().await?; + assert_eq!( + vault_secrets.len(), + secrets.len(), + "should list all populated secrets" + ); + eprintln!("Listed {} secrets from Vault", vault_secrets.len()); + + // --- Import into Postgres --- + let encryption_key = make_test_key(42); + let (routing, kms) = make_routing_and_kms(encryption_key); + + let result = carbide_api_core::secrets::import_secrets( + test_pool, + &routing, + kms.as_ref(), + &vault_secrets, + carbide_api_core::secrets::ImportApproach::All, + ) + .await?; + + assert_eq!(result.imported, secrets.len() as u64); + assert_eq!(result.skipped, 0); + eprintln!("Imported {} secrets into Postgres", result.imported); + + // --- Verify all secrets are readable from Postgres --- + let pg_mgr = carbide_api_core::secrets::PostgresCredentialManager::new( + test_pool.clone(), + routing.clone(), + kms.clone(), + ); + for (key, expected_cred) in &secrets { + let actual = pg_mgr.get_credentials(key).await?; + assert_eq!( + actual.as_ref(), + Some(expected_cred), + "secret at path {:?} should match", + key.to_key_str() + ); + } + eprintln!("All {} secrets verified in Postgres", secrets.len()); + + // --- Re-import with MissingOnly — should be a noop --- + let result2 = carbide_api_core::secrets::import_secrets( + test_pool, + &routing, + kms.as_ref(), + &vault_secrets, + carbide_api_core::secrets::ImportApproach::MissingOnly, + ) + .await?; + + assert_eq!(result2.imported, 0, "re-import should not import anything"); + assert_eq!( + result2.skipped, + secrets.len() as u64, + "re-import should skip all" + ); + eprintln!("Re-import was a noop (skipped {})", result2.skipped); + + // --- Verify marker --- + carbide_api_core::secrets::mark_vault_import_complete(test_pool, &routing, kms.as_ref()) + .await?; + assert!( + carbide_api_core::secrets::is_vault_import_complete(test_pool).await?, + "import marker should be set" + ); + eprintln!("Import marker verified"); + + Ok(()) +} diff --git a/crates/api-model/src/lib.rs b/crates/api-model/src/lib.rs index b4d16eaa0c..a3ff6e8f1d 100644 --- a/crates/api-model/src/lib.rs +++ b/crates/api-model/src/lib.rs @@ -87,6 +87,7 @@ pub mod rack_type; pub mod redfish; pub mod resource_pool; pub mod route_server; +pub mod secrets; pub mod site_explorer; pub mod sku; pub mod spx_partition; diff --git a/crates/api-model/src/secrets.rs b/crates/api-model/src/secrets.rs new file mode 100644 index 0000000000..ad4dddc2e4 --- /dev/null +++ b/crates/api-model/src/secrets.rs @@ -0,0 +1,36 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use carbide_uuid::secret::SecretId; +use chrono::{DateTime, Utc}; + +/// One row of the `secrets` table: an envelope-encrypted credential value +/// plus its wrapped DEK. `seq` is the journal order -- higher means written +/// later. Decryption lives in `carbide::secrets`; this type only moves the +/// columns. +#[derive(sqlx::FromRow)] +pub struct SecretRow { + pub secret_id: SecretId, + pub seq: i64, + pub path: String, + pub encrypted_value: Vec, + pub nonce: Vec, + pub kek_id: String, + pub created_at: DateTime, + pub encrypted_dek: Vec, + pub dek_nonce: Vec, +} diff --git a/crates/kms-provider/Cargo.toml b/crates/kms-provider/Cargo.toml index c8e6959843..cb3411589f 100644 --- a/crates/kms-provider/Cargo.toml +++ b/crates/kms-provider/Cargo.toml @@ -34,6 +34,7 @@ serde = { features = ["derive"], workspace = true } sha2 = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } +tokio-util = { workspace = true } tracing = { workspace = true } vaultrs = { workspace = true } zeroize = { workspace = true } diff --git a/crates/kms-provider/src/crypto.rs b/crates/kms-provider/src/crypto.rs index 4679d634bd..c8422cd2f4 100644 --- a/crates/kms-provider/src/crypto.rs +++ b/crates/kms-provider/src/crypto.rs @@ -15,32 +15,50 @@ * limitations under the License. */ -use aes_gcm::aead::Aead; +use aes_gcm::aead::{Aead, Payload}; use aes_gcm::{Aes256Gcm, KeyInit, Nonce}; -use sha2::{Digest, Sha256}; use crate::KmsError; -/// NONCE_LEN is the byte length of an AES-256-GCM -/// nonce. +/// The byte length of an AES-256-GCM nonce. const NONCE_LEN: usize = 12; -/// encrypt encrypts plaintext using AES-256-GCM -/// with a random 12-byte nonce. Returns -/// (ciphertext, nonce) on success. -pub fn encrypt(key: &[u8; 32], plaintext: &[u8]) -> Result<(Vec, Vec), KmsError> { +/// Encrypt plaintext with AES-256-GCM under a random 12-byte nonce, binding +/// `aad` into the authentication tag. Returns `(ciphertext, nonce)`. +/// +/// The associated data is authenticated but not encrypted: decryption fails +/// unless the same bytes are presented again. Callers use this to tie a +/// ciphertext to its storage location (the secrets table passes the row's +/// `path`), so a ciphertext copied onto another row will not decrypt. Pass +/// `b""` when there is no context to bind, which is what DEK wrapping does. +pub fn encrypt( + key: &[u8; 32], + plaintext: &[u8], + aad: &[u8], +) -> Result<(Vec, Vec), KmsError> { let cipher = Aes256Gcm::new(key.into()); let nonce_bytes: [u8; NONCE_LEN] = rand::random(); let nonce = Nonce::from_slice(&nonce_bytes); let ciphertext = cipher - .encrypt(nonce, plaintext) + .encrypt( + nonce, + Payload { + msg: plaintext, + aad, + }, + ) .map_err(|_| KmsError::EncryptionFailed("AES-256-GCM encryption failed".to_string()))?; Ok((ciphertext, nonce_bytes.to_vec())) } -/// decrypt decrypts AES-256-GCM ciphertext using -/// the provided key and nonce. -pub fn decrypt(key: &[u8; 32], nonce: &[u8], ciphertext: &[u8]) -> Result, KmsError> { +/// Decrypt AES-256-GCM ciphertext. The key, nonce, and `aad` must all match +/// what [`encrypt`] was given, or decryption fails. +pub fn decrypt( + key: &[u8; 32], + nonce: &[u8], + ciphertext: &[u8], + aad: &[u8], +) -> Result, KmsError> { if nonce.len() != NONCE_LEN { return Err(KmsError::DecryptionFailed(format!( "invalid nonce length: expected {NONCE_LEN} bytes, got {}", @@ -50,18 +68,16 @@ pub fn decrypt(key: &[u8; 32], nonce: &[u8], ciphertext: &[u8]) -> Result String { - let hash = Sha256::digest(key); - hex::encode(&hash[..8]) -} - #[cfg(test)] mod tests { use super::*; @@ -82,15 +98,14 @@ mod tests { key } - // Verifies that encrypt then decrypt produces - // the original plaintext. + // Verifies that encrypt then decrypt produces the original plaintext. #[test] fn encrypt_decrypt_round_trip() { let key = test_key(); let plaintext = b"hello, secrets!"; - let (ciphertext, nonce) = encrypt(&key, plaintext).expect("encrypt"); - let decrypted = decrypt(&key, &nonce, &ciphertext).expect("decrypt"); + let (ciphertext, nonce) = encrypt(&key, plaintext, b"").expect("encrypt"); + let decrypted = decrypt(&key, &nonce, &ciphertext, b"").expect("decrypt"); assert_eq!(decrypted, plaintext); } @@ -102,55 +117,51 @@ mod tests { let wrong_key = other_key(); let plaintext = b"sensitive data"; - let (ciphertext, nonce) = encrypt(&key, plaintext).expect("encrypt"); - let result = decrypt(&wrong_key, &nonce, &ciphertext); + let (ciphertext, nonce) = encrypt(&key, plaintext, b"").expect("encrypt"); + let result = decrypt(&wrong_key, &nonce, &ciphertext, b""); assert!(result.is_err()); } - // Verifies that encrypting the same plaintext - // twice produces different ciphertext. + // Verifies that encrypting the same plaintext twice produces different + // ciphertext (a new random nonce every call). #[test] fn different_nonces_produce_different_ciphertext() { let key = test_key(); let plaintext = b"same plaintext"; - let (ct1, nonce1) = encrypt(&key, plaintext).expect("encrypt 1"); - let (ct2, nonce2) = encrypt(&key, plaintext).expect("encrypt 2"); + let (ct1, nonce1) = encrypt(&key, plaintext, b"").expect("encrypt 1"); + let (ct2, nonce2) = encrypt(&key, plaintext, b"").expect("encrypt 2"); assert_ne!(nonce1, nonce2); assert_ne!(ct1, ct2); } - // Verifies that an invalid nonce length returns - // an error. + // Verifies that an invalid nonce length returns an error. #[test] fn invalid_nonce_length_errors() { let key = test_key(); - let result = decrypt(&key, &[0u8; 11], &[]); + let result = decrypt(&key, &[0u8; 11], &[], b""); assert!(result.is_err()); } - // Verifies that derive_key_id is deterministic. + // Verifies that the associated data is bound into the ciphertext: the + // same bytes decrypt it, different bytes do not. #[test] - fn derive_key_id_is_deterministic() { + fn aad_mismatch_fails_decryption() { let key = test_key(); - assert_eq!(derive_key_id(&key), derive_key_id(&key)); - } + let plaintext = b"bound to a path"; - // Verifies that different keys produce different - // key_ids. - #[test] - fn derive_key_id_differs_for_different_keys() { - assert_ne!(derive_key_id(&test_key()), derive_key_id(&other_key())); - } + let (ciphertext, nonce) = + encrypt(&key, plaintext, b"machines/bmc/a/root").expect("encrypt"); - // Verifies that derive_key_id produces 16 hex - // characters. - #[test] - fn derive_key_id_is_16_hex_chars() { - let id = derive_key_id(&test_key()); - assert_eq!(id.len(), 16); - assert!(id.chars().all(|c| c.is_ascii_hexdigit())); + let ok = decrypt(&key, &nonce, &ciphertext, b"machines/bmc/a/root").expect("same aad"); + assert_eq!(ok, plaintext); + + let swapped = decrypt(&key, &nonce, &ciphertext, b"machines/bmc/b/root"); + assert!( + swapped.is_err(), + "ciphertext moved to another path must not decrypt" + ); } } diff --git a/crates/kms-provider/src/lib.rs b/crates/kms-provider/src/lib.rs index 594cbd1ee6..3ee529ca96 100644 --- a/crates/kms-provider/src/lib.rs +++ b/crates/kms-provider/src/lib.rs @@ -81,6 +81,11 @@ pub trait KmsBackend: Send + Sync { /// can decrypt DEKs wrapped by the given kek_id. fn can_decrypt_kek(&self, kek_id: &str) -> bool; + /// kek_ids returns every kek_id this backend answers for. Startup + /// validation uses this to reject a kek_id configured in more than one + /// provider, which would make unwraps depend on provider order. + fn kek_ids(&self) -> Vec; + /// generate_and_wrap_dek generates a fresh DEK and /// wraps it in a single operation. The default /// generates locally and delegates to encrypt_dek. diff --git a/crates/kms-provider/src/providers/integrated.rs b/crates/kms-provider/src/providers/integrated.rs index 2aacbf1871..70c4e5b3e0 100644 --- a/crates/kms-provider/src/providers/integrated.rs +++ b/crates/kms-provider/src/providers/integrated.rs @@ -25,18 +25,18 @@ use zeroize::{Zeroize, Zeroizing}; use crate::{EncryptedDek, KmsBackend, KmsError, crypto}; -/// KeySource describes where to load a symmetric -/// encryption key from. +/// Where to load a base64-encoded 256-bit key from. +/// +/// Both sources keep key material out of the carbide config file itself, +/// which is Debug-logged at startup and served by the web debug page. There +/// is deliberately no inline-value variant for the same reason. #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] #[serde(untagged)] pub enum KeySource { - /// Env loads the base64-encoded key from an environment - /// variable. + /// Load the key from an environment variable. Env { env: String }, - /// File loads the base64-encoded key from a file path. + /// Load the key from a file path. File { file: PathBuf }, - /// Value contains the base64-encoded key directly. - Value { value: String }, } /// IntegratedKmsProvider implements KmsBackend using @@ -55,8 +55,7 @@ impl Drop for IntegratedKmsProvider { } } -/// decode_key decodes a base64-encoded 256-bit key. -/// The intermediate buffer is zeroized. +/// Decode a base64-encoded 256-bit key. The intermediate buffer is zeroized. fn decode_key(encoded: &str) -> Result<[u8; 32], KmsError> { let mut bytes = BASE64 .decode(encoded.trim()) @@ -70,23 +69,49 @@ fn decode_key(encoded: &str) -> Result<[u8; 32], KmsError> { result } -/// resolve_key_source loads a key from the given source. +/// Load a key from the given source. The base64 string read from the +/// environment or file is zeroized along with the decoded buffer. fn resolve_key_source(source: &KeySource) -> Result<[u8; 32], KmsError> { match source { KeySource::Env { env } => { - let val = std::env::var(env) - .map_err(|_| KmsError::Other(format!("environment variable {env:?} not set")))?; + let val = + Zeroizing::new(std::env::var(env).map_err(|_| { + KmsError::Other(format!("environment variable {env:?} not set")) + })?); decode_key(&val) } KeySource::File { file } => { - let val = std::fs::read_to_string(file) - .map_err(|e| KmsError::Other(format!("failed to read key file {file:?}: {e}")))?; + warn_if_key_file_is_open(file); + let val = + Zeroizing::new(std::fs::read_to_string(file).map_err(|e| { + KmsError::Other(format!("failed to read key file {file:?}: {e}")) + })?); decode_key(&val) } - KeySource::Value { value } => decode_key(value), } } +/// Warn when a key file is readable by group or other. The provider still +/// loads the key -- deployments mount these files in ways we cannot always +/// predict -- but the warning gives operators a clear signal to fix the mode. +#[cfg(unix)] +fn warn_if_key_file_is_open(file: &std::path::Path) { + use std::os::unix::fs::MetadataExt; + if let Ok(meta) = std::fs::metadata(file) { + let mode = meta.mode() & 0o077; + if mode != 0 { + tracing::warn!( + file = %file.display(), + mode = format!("{:o}", meta.mode() & 0o777), + "KEK file is readable by group/other; consider mode 0600" + ); + } + } +} + +#[cfg(not(unix))] +fn warn_if_key_file_is_open(_file: &std::path::Path) {} + impl IntegratedKmsProvider { /// IntegratedKmsProvider::from_config builds a /// provider from a map of kek_id to key source. @@ -119,7 +144,7 @@ impl KmsBackend for IntegratedKmsProvider { .keys .get(kek_id) .ok_or_else(|| KmsError::KeyNotFound(kek_id.to_string()))?; - let (ciphertext, nonce) = crypto::encrypt(kek, dek)?; + let (ciphertext, nonce) = crypto::encrypt(kek, dek, b"")?; Ok(EncryptedDek { ciphertext, nonce }) } @@ -132,7 +157,7 @@ impl KmsBackend for IntegratedKmsProvider { .keys .get(kek_id) .ok_or_else(|| KmsError::KeyNotFound(kek_id.to_string()))?; - let mut plaintext = crypto::decrypt(kek, &encrypted.nonce, &encrypted.ciphertext)?; + let mut plaintext = crypto::decrypt(kek, &encrypted.nonce, &encrypted.ciphertext, b"")?; let len = plaintext.len(); let dek: [u8; 32] = plaintext .as_slice() @@ -145,6 +170,10 @@ impl KmsBackend for IntegratedKmsProvider { fn can_decrypt_kek(&self, kek_id: &str) -> bool { self.keys.contains_key(kek_id) } + + fn kek_ids(&self) -> Vec { + self.keys.keys().cloned().collect() + } } #[cfg(test)] @@ -266,24 +295,7 @@ mod tests { assert!(provider.can_decrypt_kek("file-key")); } - // Verifies that from_config loads inline base64 values. - #[test] - fn from_config_value_source() { - let key = make_test_key(3); - let mut key_map = HashMap::new(); - key_map.insert( - "inline-key".to_string(), - KeySource::Value { - value: encode_key(&key), - }, - ); - - let provider = IntegratedKmsProvider::from_config(&key_map).expect("from_config"); - assert!(provider.can_decrypt_kek("inline-key")); - } - - // Verifies that from_config errors when no keys - // are provided. + // Verifies that from_config errors when no keys are provided. #[test] fn from_config_empty_errors() { let result = IntegratedKmsProvider::from_config(&HashMap::new()); @@ -293,29 +305,26 @@ mod tests { // Verifies that from_config errors on invalid base64. #[test] fn from_config_invalid_base64_errors() { + let dir = tempfile::tempdir().expect("tempdir"); + let file_path = dir.path().join("bad-key"); + std::fs::write(&file_path, "not-valid-base64!!!").expect("write"); + let mut key_map = HashMap::new(); - key_map.insert( - "bad-key".to_string(), - KeySource::Value { - value: "not-valid-base64!!!".to_string(), - }, - ); + key_map.insert("bad-key".to_string(), KeySource::File { file: file_path }); let result = IntegratedKmsProvider::from_config(&key_map); assert!(result.is_err()); } - // Verifies that from_config errors when a key - // is not 32 bytes. + // Verifies that from_config errors when a key is not 32 bytes. #[test] fn from_config_wrong_key_length_errors() { + let dir = tempfile::tempdir().expect("tempdir"); + let file_path = dir.path().join("short-key"); + std::fs::write(&file_path, BASE64.encode([0u8; 16])).expect("write"); + let mut key_map = HashMap::new(); - key_map.insert( - "short-key".to_string(), - KeySource::Value { - value: BASE64.encode([0u8; 16]), - }, - ); + key_map.insert("short-key".to_string(), KeySource::File { file: file_path }); let result = IntegratedKmsProvider::from_config(&key_map); assert!(result.is_err()); diff --git a/crates/kms-provider/src/providers/multi.rs b/crates/kms-provider/src/providers/multi.rs index 7fcf363683..125b1a81e1 100644 --- a/crates/kms-provider/src/providers/multi.rs +++ b/crates/kms-provider/src/providers/multi.rs @@ -76,6 +76,10 @@ impl KmsBackend for MultiKmsProvider { self.providers.iter().any(|p| p.can_decrypt_kek(kek_id)) } + fn kek_ids(&self) -> Vec { + self.providers.iter().flat_map(|p| p.kek_ids()).collect() + } + async fn generate_and_wrap_dek( &self, kek_id: &str, diff --git a/crates/kms-provider/src/providers/transit.rs b/crates/kms-provider/src/providers/transit.rs index 199ed69a1f..8dfd159b47 100644 --- a/crates/kms-provider/src/providers/transit.rs +++ b/crates/kms-provider/src/providers/transit.rs @@ -67,19 +67,30 @@ impl TransitKmsProvider { } } - /// start_token_renewal spawns a background task - /// that periodically renews the Vault token. Right - /// now it's just renewing at 90% (0.9) of the lease - /// duration. - pub fn start_token_renewal(&self) -> tokio::task::JoinHandle<()> { + /// run_token_renewal returns a future that renews + /// the Vault token at 90% (0.9) of each lease + /// duration until `cancel` fires. The caller spawns + /// it -- typically onto the process JoinSet, so + /// shutdown actually stops it. + pub fn run_token_renewal( + &self, + cancel: tokio_util::sync::CancellationToken, + ) -> impl Future + Send + 'static { let client = self.client.clone(); - tokio::spawn(async move { - // Initial lookup to get the token's TTL and renewability. - let info = match vaultrs::token::lookup_self(client.as_ref()).await { - Ok(info) => info, - Err(e) => { - tracing::warn!("failed to look up Transit KMS token: {e}"); - return; + async move { + // Look up the token's TTL and renewability, retrying until it + // answers -- a vault blip while carbide-api boots must not + // disable renewal for the life of the process. + let info = loop { + match vaultrs::token::lookup_self(client.as_ref()).await { + Ok(info) => break info, + Err(e) => { + tracing::warn!("failed to look up Transit KMS token, retrying: {e}"); + tokio::select! { + _ = cancel.cancelled() => return, + _ = tokio::time::sleep(Duration::from_secs(30)) => {} + } + } } }; @@ -94,7 +105,10 @@ impl TransitKmsProvider { sleep_secs = next_renewal.as_secs(), "scheduling Transit KMS token renewal" ); - tokio::time::sleep(next_renewal).await; + tokio::select! { + _ = cancel.cancelled() => return, + _ = tokio::time::sleep(next_renewal) => {} + } match vaultrs::token::renew_self(client.as_ref(), None).await { Ok(renewed) => { @@ -112,14 +126,16 @@ impl TransitKmsProvider { } } } - }) + } } } #[async_trait] impl KmsBackend for TransitKmsProvider { async fn encrypt_dek(&self, kek_id: &str, dek: &[u8; 32]) -> Result { - let plaintext_b64 = BASE64.encode(dek); + // The base64 string is another copy of the DEK; zeroize it like the + // decoded buffers. + let plaintext_b64 = Zeroizing::new(BASE64.encode(dek)); let response = transit::data::encrypt( self.client.as_ref(), &self.transit_mount, @@ -130,9 +146,9 @@ impl KmsBackend for TransitKmsProvider { .await .map_err(|e| KmsError::EncryptionFailed(format!("vault transit encrypt: {e}")))?; - // Vault Transit returns ciphertext as a string like "vault:v1:". - // Store the entire string as bytes, and then just 'll pass it back verbatim - // to decrypt. + // Vault Transit returns ciphertext as a string like + // "vault:v1:". Store the entire string as bytes and pass it + // back verbatim on decrypt. Ok(EncryptedDek { ciphertext: response.ciphertext.into_bytes(), nonce: vec![], // Transit manages nonces internally. @@ -144,22 +160,24 @@ impl KmsBackend for TransitKmsProvider { kek_id: &str, encrypted: &EncryptedDek, ) -> Result, KmsError> { - let ciphertext_str = String::from_utf8(encrypted.ciphertext.clone()) + let ciphertext_str = std::str::from_utf8(&encrypted.ciphertext) .map_err(|_| KmsError::DecryptionFailed("invalid ciphertext encoding".to_string()))?; let response = transit::data::decrypt( self.client.as_ref(), &self.transit_mount, kek_id, - &ciphertext_str, + ciphertext_str, None, ) .await .map_err(|e| KmsError::DecryptionFailed(format!("vault transit decrypt: {e}")))?; - // Vault returns base64-encoded plaintext. + // Vault returns base64-encoded plaintext: one more copy of the DEK, + // zeroized along with the decoded buffer. + let plaintext_b64 = Zeroizing::new(response.plaintext); let mut decoded = BASE64 - .decode(&response.plaintext) + .decode(plaintext_b64.as_bytes()) .map_err(|e| KmsError::DecryptionFailed(format!("invalid base64 from vault: {e}")))?; let len = decoded.len(); let dek: [u8; 32] = decoded @@ -174,6 +192,10 @@ impl KmsBackend for TransitKmsProvider { self.known_keys.iter().any(|k| k == kek_id) } + fn kek_ids(&self) -> Vec { + self.known_keys.clone() + } + async fn generate_and_wrap_dek( &self, kek_id: &str, @@ -188,12 +210,12 @@ impl KmsBackend for TransitKmsProvider { .await .map_err(|e| KmsError::EncryptionFailed(format!("vault transit generate data key: {e}")))?; - let plaintext_b64 = response.plaintext.ok_or_else(|| { + let plaintext_b64 = Zeroizing::new(response.plaintext.ok_or_else(|| { KmsError::Other("vault returned no plaintext for data key".to_string()) - })?; + })?); let mut decoded = BASE64 - .decode(&plaintext_b64) + .decode(plaintext_b64.as_bytes()) .map_err(|e| KmsError::Other(format!("invalid base64 from vault: {e}")))?; let len = decoded.len(); let dek: [u8; 32] = decoded diff --git a/crates/rpc/proto/forge.proto b/crates/rpc/proto/forge.proto index 6c33dd7235..596bbef2c6 100644 --- a/crates/rpc/proto/forge.proto +++ b/crates/rpc/proto/forge.proto @@ -942,6 +942,9 @@ service Forge { // Updates the cached_url field of named artifacts in-place, leaving all other artifact fields // and OS definition fields (including ipxe_definition_hash) unchanged. rpc UpdateOperatingSystemCachableIpxeTemplateArtifacts(UpdateOperatingSystemIpxeTemplateArtifactRequest) returns (IpxeTemplateArtifactList); + + // Secrets management + rpc ReWrapSecrets(ReWrapSecretsRequest) returns (ReWrapSecretsResponse); } // Indicates the lifecycle state of a resource that is controlled by a state controller @@ -8543,3 +8546,24 @@ message HostRepresentorInterceptBridging { string bridge = 1; string patch_port = 2; } + +message ReWrapSecretsRequest { + // Rows scanned per batch during the re-wrap walk. + // Unset: the server picks a sensible default (100). + // Out of range: the server clamps to [1, 10000]; in + // particular 0 does not mean "scan nothing". Clients + // should not rely on sending 0 or very large values. + optional uint32 batch_size = 1; +} + +message ReWrapSecretsResponse { + // Journal rows whose DEK was re-wrapped to the + // routed KEK. + uint64 re_wrapped = 1; + // Journal rows already wrapped by the routed KEK. + uint64 already_current = 2; + // Journal rows still wrapped by a KEK outside the + // routing config after the walk. Zero means every + // unrouted KEK can be retired. + uint64 stale_remaining = 3; +} diff --git a/crates/secrets/src/forge_vault.rs b/crates/secrets/src/forge_vault.rs index ab1cb5ce23..e1fe2e2e19 100644 --- a/crates/secrets/src/forge_vault.rs +++ b/crates/secrets/src/forge_vault.rs @@ -765,11 +765,27 @@ impl CertificateProvider for ForgeVaultClient { } } +/// How a bulk enumeration treats vault errors other than 404 (which always +/// just means "nothing here"). +#[derive(Clone, Copy, PartialEq, Eq)] +enum EnumerationMode { + /// Warn and keep going. Fine for diagnostics, where a partial answer + /// beats none. + BestEffort, + /// Fail the whole enumeration. Required when the caller will act on + /// the result as if it were complete -- the one-time import writes a + /// permanent completion marker, so a silently dropped subtree would + /// become silently lost credentials. + Strict, +} + impl ForgeVaultClient { /// list_secrets returns all secret paths in the /// KV mount. pub async fn list_secrets(&self) -> Result, SecretsError> { - let paths = self.list_secrets_for_path("").await?; + let paths = self + .list_secrets_for_path("", EnumerationMode::BestEffort) + .await?; tracing::info!(count = paths.len(), "listed all vault secret paths"); Ok(paths) } @@ -780,7 +796,9 @@ impl ForgeVaultClient { &self, prefix: &crate::credentials::CredentialPrefix, ) -> Result, SecretsError> { - let paths = self.list_secrets_for_path(prefix.as_str()).await?; + let paths = self + .list_secrets_for_path(prefix.as_str(), EnumerationMode::BestEffort) + .await?; tracing::info!( prefix = prefix.as_str(), count = paths.len(), @@ -789,12 +807,12 @@ impl ForgeVaultClient { Ok(paths) } - /// list_secrets_for_path recursively lists all - /// secret paths under the given path prefix in - /// the KV mount. - pub async fn list_secrets_for_path( + /// list_secrets_for_path recursively lists all secret paths under the + /// given path prefix in the KV mount. + async fn list_secrets_for_path( &self, path_prefix: &str, + mode: EnumerationMode, ) -> Result, SecretsError> { let vault_client = self.vault_client().await?; let mount = &self.vault_client_config.kv_mount_location; @@ -806,10 +824,16 @@ impl ForgeVaultClient { let entries = match kv2::list(vault_client.deref(), mount, &dir).await { Ok(e) => e, Err(ClientError::APIError { code: 404, .. }) => continue, + Err(e) if mode == EnumerationMode::Strict => { + return Err(SecretsError::GenericError(eyre!( + "failed to list vault path {dir:?}: {e}" + ))); + } Err(e) => { tracing::warn!( prefix = %dir, - "failed to list vault path: {e}" + error = %e, + "failed to list vault path" ); continue; } @@ -837,11 +861,25 @@ impl ForgeVaultClient { Ok(paths) } - /// get_secrets returns all secrets in the KV - /// mount (paths + credentials). + /// get_secrets returns all secrets in the KV mount (paths plus + /// credentials), skipping unreadable entries with a warning. pub async fn get_secrets(&self) -> Result, SecretsError> { - let paths = self.list_secrets().await?; - self.read_secrets(&paths).await + let paths = self + .list_secrets_for_path("", EnumerationMode::BestEffort) + .await?; + self.read_secrets(&paths, EnumerationMode::BestEffort).await + } + + /// get_secrets_strict returns all secrets in the KV mount, failing on + /// the first list or read error instead of skipping. The one-time + /// Postgres import uses this so a vault hiccup aborts the import -- + /// and leaves the completion marker unwritten -- rather than quietly + /// importing a subset. + pub async fn get_secrets_strict(&self) -> Result, SecretsError> { + let paths = self + .list_secrets_for_path("", EnumerationMode::Strict) + .await?; + self.read_secrets(&paths, EnumerationMode::Strict).await } /// get_secrets_for_prefix returns all secrets @@ -850,8 +888,10 @@ impl ForgeVaultClient { &self, prefix: &crate::credentials::CredentialPrefix, ) -> Result, SecretsError> { - let paths = self.list_secrets_for_prefix(prefix).await?; - self.read_secrets(&paths).await + let paths = self + .list_secrets_for_path(prefix.as_str(), EnumerationMode::BestEffort) + .await?; + self.read_secrets(&paths, EnumerationMode::BestEffort).await } /// get_secrets_for_path returns all secrets under @@ -860,16 +900,19 @@ impl ForgeVaultClient { &self, path_prefix: &str, ) -> Result, SecretsError> { - let paths = self.list_secrets_for_path(path_prefix).await?; - self.read_secrets(&paths).await + let paths = self + .list_secrets_for_path(path_prefix, EnumerationMode::BestEffort) + .await?; + self.read_secrets(&paths, EnumerationMode::BestEffort).await } - /// read_secrets reads credentials from vault for - /// each path. Skips 404s and logs warnings on - /// other errors. + /// read_secrets reads credentials from vault for each path. 404s are + /// always skipped (deleted between list and read); other errors follow + /// the enumeration mode. async fn read_secrets( &self, paths: &[String], + mode: EnumerationMode, ) -> Result, SecretsError> { let vault_client = self.vault_client().await?; let mount = &self.vault_client_config.kv_mount_location; @@ -886,10 +929,16 @@ impl ForgeVaultClient { "vault secret not found" ); } + Err(e) if mode == EnumerationMode::Strict => { + return Err(SecretsError::GenericError(eyre!( + "failed to read vault secret {path:?}: {e}" + ))); + } Err(e) => { tracing::warn!( path = %path, - "failed to read: {e}" + error = %e, + "failed to read vault secret" ); } } @@ -1037,6 +1086,36 @@ pub fn create_vault_client( Ok(Arc::new(forge_vault_client)) } +/// Build raw vaultrs client settings for a separate vault consumer (the +/// Transit KMS provider), with the same address, CA trust, and timeout that +/// `ForgeVaultClient` itself connects with. Without the CA wiring, a +/// vaultrs client only trusts public roots and fails TLS against a +/// site-CA-signed vault. +/// +/// Authentication is NOT at parity with `ForgeVaultClient`: this requires a +/// static vault token in the config and does not support the Kubernetes +/// service-account login flow. Deployments using SA auth cannot configure a +/// transit KMS provider until that lands. +pub fn create_raw_vault_client_settings( + vault_config: &VaultConfig, +) -> eyre::Result { + let configured_ca_path = vault_config + .vault_cacert() + .unwrap_or_else(|_| DEFAULT_VAULT_CA_PATH.to_string()); + let ca_path = resolve_vault_root_ca_path(configured_ca_path.as_str())?; + + let mut builder = VaultClientSettingsBuilder::default(); + builder + .token(vault_config.token()?) + .address(vault_config.address()?) + .timeout(Some(Duration::from_secs(60))) + .ca_certs(vec![ca_path]) + .verify(true); + builder + .build() + .map_err(|e| eyre!("vault client settings: {e}")) +} + #[cfg(test)] mod tests { use base64::Engine; diff --git a/crates/secrets/src/lib.rs b/crates/secrets/src/lib.rs index ccf3495537..c086fd54b5 100644 --- a/crates/secrets/src/lib.rs +++ b/crates/secrets/src/lib.rs @@ -20,9 +20,13 @@ use std::sync::Arc; use opentelemetry::metrics::Meter; pub use crate::chained_reader::ChainedCredentialReader; -/// Exposed for `CertificateProvider` usage only. Credential operations should go +/// Direct vault access for the narrow cases that need it: `CertificateProvider` +/// (PKI), and the Transit KMS provider, which builds its own raw vault client +/// via `create_raw_vault_client_settings`. Credential operations should go /// through `create_credential_manager` instead of using the vault client directly. -pub use crate::forge_vault::{ForgeVaultClient, VaultConfig, create_vault_client}; +pub use crate::forge_vault::{ + ForgeVaultClient, VaultConfig, create_raw_vault_client_settings, create_vault_client, +}; pub use crate::local_credentials::{ CredentialSnapshot, EnvCredentialsConfig, FileCredentialsConfig, MachineIdentityConfig, UsernamePassword, diff --git a/crates/uuid/src/lib.rs b/crates/uuid/src/lib.rs index 8bb8287ef2..ab3cfec6d5 100644 --- a/crates/uuid/src/lib.rs +++ b/crates/uuid/src/lib.rs @@ -33,6 +33,7 @@ pub mod nvlink; pub mod operating_system; pub mod power_shelf; pub mod rack; +pub mod secret; pub mod spx; pub mod switch; pub mod typed_uuids; diff --git a/crates/uuid/src/secret.rs b/crates/uuid/src/secret.rs new file mode 100644 index 0000000000..0627f47083 --- /dev/null +++ b/crates/uuid/src/secret.rs @@ -0,0 +1,36 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use crate::typed_uuids::{TypedUuid, UuidSubtype}; + +/// SecretFlavor is the marker type for secret UUIDs. +pub struct SecretFlavor; + +impl UuidSubtype for SecretFlavor { + const TYPE_NAME: &'static str = "SecretId"; + const DB_COLUMN_NAME: &'static str = "secret_id"; +} + +/// SecretId uniquely identifies a row in the secrets table. +pub type SecretId = TypedUuid; + +#[cfg(test)] +mod tests { + use super::*; + + crate::typed_uuid_tests!(SecretId, "SecretId", "secret_id"); +}