From f5fceb475cd88e3a9f49a061067598271b5b47eb Mon Sep 17 00:00:00 2001 From: Anton Liashkevich Date: Tue, 21 Jul 2026 19:28:04 -0400 Subject: [PATCH 1/3] feat(grpc): encrypt the control channel with server-side TLS --- .gitignore | 1 + Cargo.lock | 1 + README.md | 21 ++ members/nullnet-client/src/env.rs | 6 + members/nullnet-client/src/main.rs | 9 +- members/nullnet-grpc-lib/Cargo.toml | 5 +- .../src/control_tls_verifier.rs | 96 +++++++++ members/nullnet-grpc-lib/src/lib.rs | 23 ++- members/nullnet-proxy/src/env.rs | 6 + members/nullnet-proxy/src/nullnet_proxy.rs | 13 +- members/nullnet-server/src/grpc_tls.rs | 185 ++++++++++++++++++ members/nullnet-server/src/main.rs | 16 +- 12 files changed, 356 insertions(+), 26 deletions(-) create mode 100644 members/nullnet-grpc-lib/src/control_tls_verifier.rs create mode 100644 members/nullnet-server/src/grpc_tls.rs diff --git a/.gitignore b/.gitignore index 5d7d82a0..d73a10dc 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ peers.txt members/nullnet-server/ui/node_modules/ members/nullnet-server/ui/dist/ members/nullnet-server/certs +members/nullnet-server/grpc-tls diff --git a/Cargo.lock b/Cargo.lock index fcfd7a0c..b2e07aeb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2826,6 +2826,7 @@ dependencies = [ "prost", "serde", "tokio", + "tokio-rustls 0.26.4", "tonic", "tonic-prost", "tonic-prost-build", diff --git a/README.md b/README.md index e992ef72..d095bd12 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,21 @@ The repository should be cloned under `/root` so the provided `setup-*.sh` scrip CERT_RENEWAL_DNS_PROPAGATION_SECS=30 # wait after writing the TXT record ``` +- the gRPC control channel itself (nullnet-client/nullnet-proxy ↔ nullnet-server) is TLS-only, + authenticated by a private CA. On first boot the server generates its own CA + (`members/nullnet-server/grpc-tls/ca-cert.pem` + `ca-key.pem`, created once and never + regenerated) and signs its leaf cert with it. Copy `ca-cert.pem` to every client/proxy host and + set `CONTROL_SERVICE_CA_CERT` there (see below) — it's **required**: clients pin the channel to + that CA and do full standard chain validation, so only a leaf actually signed by it is accepted. + Because clients trust the stable CA root rather than the leaf, rotating the leaf later needs no + client-side changes. Client authentication (mTLS) remains a further follow-up. + + Validation includes hostname matching, so the leaf's SAN must cover whatever host/IP clients use + as `CONTROL_SERVICE_ADDR`. It's derived in this order: `CONTROL_SERVICE_TLS_SAN` + (comma-separated, if set) → the server's own `CONTROL_SERVICE_ADDR` (if set — often already the + same address clients are told to connect to) → `localhost`. Set `CONTROL_SERVICE_TLS_SAN` + explicitly if the server's address isn't in its own `.env` or differs from what clients use. + - service configuration is split per **stack** — one TOML file per stack under `members/nullnet-server/services/`. The filename (minus `.toml`) is the stack name. For example, to define a stack called `my-app`, create `services/my-app.toml`: @@ -182,7 +197,10 @@ The repository should be cloned under `/root` so the provided `setup-*.sh` scrip ``` CONTROL_SERVICE_ADDR=192.168.1.100 CONTROL_SERVICE_PORT=50051 + CONTROL_SERVICE_CA_CERT=./ca-cert.pem ``` + `CONTROL_SERVICE_CA_CERT` is **required** — point it at a copy of the server's own + `grpc-tls/ca-cert.pem` (see the server section above) to pin and authenticate the control channel. - run the project as a daemon (from the repo root) ``` @@ -205,7 +223,10 @@ The repository should be cloned under `/root` so the provided `setup-*.sh` scrip ``` CONTROL_SERVICE_ADDR=192.168.1.100 CONTROL_SERVICE_PORT=50051 + CONTROL_SERVICE_CA_CERT=./ca-cert.pem ``` + `CONTROL_SERVICE_CA_CERT` is **required** — point it at a copy of the server's own + `grpc-tls/ca-cert.pem` (see the server section above) to pin and authenticate the control channel. > **⚠️ The client attaches a default-deny eBPF firewall to the uplink NIC on startup.** It permits > only the nullnet control plane (gRPC to the server), data plane (VXLAN to peers), established diff --git a/members/nullnet-client/src/env.rs b/members/nullnet-client/src/env.rs index 20c599ea..16a981f6 100644 --- a/members/nullnet-client/src/env.rs +++ b/members/nullnet-client/src/env.rs @@ -13,3 +13,9 @@ pub static CONTROL_SERVICE_PORT: std::sync::LazyLock = std::sync::LazyLock: str.parse().unwrap_or(50051) }); + +/// Path to the control server's private CA root (its `grpc-tls/ca-cert.pem` +/// — generated automatically on the server, copy it here). Required: pins +/// the control channel to that CA for full standard chain validation. +pub static CONTROL_SERVICE_CA_CERT: std::sync::LazyLock> = + std::sync::LazyLock::new(|| std::env::var("CONTROL_SERVICE_CA_CERT").ok()); diff --git a/members/nullnet-client/src/main.rs b/members/nullnet-client/src/main.rs index 558c83b6..855e9b84 100644 --- a/members/nullnet-client/src/main.rs +++ b/members/nullnet-client/src/main.rs @@ -5,7 +5,7 @@ use crate::commands::{ RtNetLinkHandle, cleanup_network, find_ethernet_interface, find_ethernet_ip, setup_br0, }; use crate::control_channel::control_channel; -use crate::env::{CONTROL_SERVICE_ADDR, CONTROL_SERVICE_PORT}; +use crate::env::{CONTROL_SERVICE_ADDR, CONTROL_SERVICE_CA_CERT, CONTROL_SERVICE_PORT}; use crate::forward::receive::receive; use crate::forward::send::send; use crate::host_mappings::HostMappingsState; @@ -20,6 +20,7 @@ use nullnet_grpc_lib::nullnet_grpc::{ }; use nullnet_liberror::{Error, ErrorHandler, Location, location}; use std::collections::HashMap; +use std::path::Path; use std::sync::Arc; use std::time::Duration; use std::{panic, process}; @@ -280,8 +281,12 @@ fn resolve_server_ip() -> Option { async fn grpc_init() -> Result { let host = CONTROL_SERVICE_ADDR.to_string(); let port = *CONTROL_SERVICE_PORT; + let ca_cert = CONTROL_SERVICE_CA_CERT + .as_deref() + .ok_or("'CONTROL_SERVICE_CA_CERT' environment variable must be set") + .handle_err(location!())?; - let server = NullnetGrpcInterface::new(&host, port, false) + let server = NullnetGrpcInterface::new(&host, port, Path::new(ca_cert)) .await .handle_err(location!())?; diff --git a/members/nullnet-grpc-lib/Cargo.toml b/members/nullnet-grpc-lib/Cargo.toml index 584b9a10..a6ad406c 100644 --- a/members/nullnet-grpc-lib/Cargo.toml +++ b/members/nullnet-grpc-lib/Cargo.toml @@ -9,10 +9,11 @@ keywords = ["firewall", "network", "application", "centralized", "monitor"] categories = ["network-programming"] [dependencies] -tonic = { workspace = true, features = ["_tls-any", "tls-native-roots"] } +tonic = { workspace = true, features = ["_tls-any"] } prost.workspace = true tonic-prost.workspace = true -tokio.workspace = true +tokio = { workspace = true, features = ["fs"] } +tokio-rustls = "0.26" serde = { workspace = true, features = ["derive"] } [build-dependencies] diff --git a/members/nullnet-grpc-lib/src/control_tls_verifier.rs b/members/nullnet-grpc-lib/src/control_tls_verifier.rs new file mode 100644 index 00000000..414c365c --- /dev/null +++ b/members/nullnet-grpc-lib/src/control_tls_verifier.rs @@ -0,0 +1,96 @@ +//! Certificate verifier for the gRPC control channel: standard WebPKI chain +//! validation against a single pinned CA root — the server's own private CA +//! (see `nullnet-server`'s `grpc_tls.rs`). Only a leaf actually signed by +//! that CA, for the right host, passes. +use std::sync::Arc; +use tokio_rustls::rustls::client::WebPkiServerVerifier; +use tokio_rustls::rustls::client::danger::{ + HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier, +}; +use tokio_rustls::rustls::crypto::{ + CryptoProvider, verify_tls12_signature, verify_tls13_signature, +}; +use tokio_rustls::rustls::pki_types::pem::PemObject; +use tokio_rustls::rustls::pki_types::{CertificateDer, ServerName, UnixTime}; +use tokio_rustls::rustls::{DigitallySignedStruct, Error, RootCertStore, SignatureScheme}; + +/// The process's default crypto provider, falling back to aws-lc-rs (the +/// backend `tokio-rustls`'s default features pull in) if nothing has +/// installed one yet. +fn crypto_provider() -> Arc { + CryptoProvider::get_default() + .cloned() + .unwrap_or_else(|| Arc::new(tokio_rustls::rustls::crypto::aws_lc_rs::default_provider())) +} + +#[derive(Debug)] +pub(crate) struct PinnedCa { + webpki: Arc, + provider: Arc, +} + +impl PinnedCa { + /// `ca_cert_pem`: the server's private CA root, PEM-encoded (its + /// `grpc-tls/ca-cert.pem`) — the only cert this verifier trusts. + pub(crate) fn verifier(ca_cert_pem: &[u8]) -> Result, String> { + let provider = crypto_provider(); + + let ca_cert = CertificateDer::from_pem_slice(ca_cert_pem).map_err(|e| e.to_string())?; + let mut roots = RootCertStore::empty(); + roots.add(ca_cert).map_err(|e| e.to_string())?; + + let webpki = WebPkiServerVerifier::builder_with_provider(Arc::new(roots), provider.clone()) + .build() + .map_err(|e| e.to_string())?; + + Ok(Arc::new(Self { webpki, provider })) + } +} + +impl ServerCertVerifier for PinnedCa { + fn verify_server_cert( + &self, + end_entity: &CertificateDer<'_>, + intermediates: &[CertificateDer<'_>], + server_name: &ServerName<'_>, + ocsp_response: &[u8], + now: UnixTime, + ) -> Result { + self.webpki + .verify_server_cert(end_entity, intermediates, server_name, ocsp_response, now) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + verify_tls12_signature( + message, + cert, + dss, + &self.provider.signature_verification_algorithms, + ) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + verify_tls13_signature( + message, + cert, + dss, + &self.provider.signature_verification_algorithms, + ) + } + + fn supported_verify_schemes(&self) -> Vec { + self.provider + .signature_verification_algorithms + .supported_schemes() + } +} diff --git a/members/nullnet-grpc-lib/src/lib.rs b/members/nullnet-grpc-lib/src/lib.rs index 84892fa6..8783d367 100644 --- a/members/nullnet-grpc-lib/src/lib.rs +++ b/members/nullnet-grpc-lib/src/lib.rs @@ -1,5 +1,7 @@ +mod control_tls_verifier; mod proto; +use crate::control_tls_verifier::PinnedCa; use crate::nullnet_grpc::nullnet_grpc_client::NullnetGrpcClient; use crate::nullnet_grpc::{ AgentEvent, BackendTriggerRequest, CertBundle, EgressDestinationEntry, EgressDestinationReport, @@ -7,6 +9,7 @@ use crate::nullnet_grpc::{ PortMappingBundle, ProxyRequest, ServiceReport, ServicesListResponse, Upstream, }; pub use proto::*; +use std::path::Path; use tokio::sync::mpsc; use tonic::Request; pub use tonic::Streaming; @@ -19,25 +22,25 @@ pub struct NullnetGrpcInterface { } impl NullnetGrpcInterface { + /// Connects over TLS, trusting *only* `ca_cert` (PEM) — the server's + /// private CA root — for full standard chain validation, including + /// hostname matching (see `control_tls_verifier::PinnedCa`). #[allow(clippy::missing_errors_doc)] - pub async fn new(host: &str, port: u16, tls: bool) -> Result { - let protocol = if tls { "https" } else { "http" }; + pub async fn new(host: &str, port: u16, ca_cert: &Path) -> Result { + let ca_cert_pem = tokio::fs::read(ca_cert).await.map_err(|e| e.to_string())?; + let verifier = PinnedCa::verifier(&ca_cert_pem)?; // Keepalive so a dead/unreachable server breaks open streams within // ~30s (PING every 10s, drop if unacked for 20s) instead of hanging. // Consumers rely on the stream erroring out to exit and be restarted. - let mut endpoint = Channel::from_shared(format!("{protocol}://{host}:{port}")) + let endpoint = Channel::from_shared(format!("https://{host}:{port}")) .map_err(|e| e.to_string())? .connect_timeout(std::time::Duration::from_secs(10)) .http2_keep_alive_interval(std::time::Duration::from_secs(10)) .keep_alive_timeout(std::time::Duration::from_secs(20)) - .keep_alive_while_idle(true); - - if tls { - endpoint = endpoint - .tls_config(ClientTlsConfig::new().with_native_roots()) - .map_err(|e| e.to_string())?; - } + .keep_alive_while_idle(true) + .tls_config_with_verifier(ClientTlsConfig::new(), verifier) + .map_err(|e| e.to_string())?; loop { if let Ok(channel) = endpoint.connect().await { diff --git a/members/nullnet-proxy/src/env.rs b/members/nullnet-proxy/src/env.rs index 20c599ea..16a981f6 100644 --- a/members/nullnet-proxy/src/env.rs +++ b/members/nullnet-proxy/src/env.rs @@ -13,3 +13,9 @@ pub static CONTROL_SERVICE_PORT: std::sync::LazyLock = std::sync::LazyLock: str.parse().unwrap_or(50051) }); + +/// Path to the control server's private CA root (its `grpc-tls/ca-cert.pem` +/// — generated automatically on the server, copy it here). Required: pins +/// the control channel to that CA for full standard chain validation. +pub static CONTROL_SERVICE_CA_CERT: std::sync::LazyLock> = + std::sync::LazyLock::new(|| std::env::var("CONTROL_SERVICE_CA_CERT").ok()); diff --git a/members/nullnet-proxy/src/nullnet_proxy.rs b/members/nullnet-proxy/src/nullnet_proxy.rs index a7753503..e9e3af2e 100644 --- a/members/nullnet-proxy/src/nullnet_proxy.rs +++ b/members/nullnet-proxy/src/nullnet_proxy.rs @@ -1,4 +1,4 @@ -use crate::env::{CONTROL_SERVICE_ADDR, CONTROL_SERVICE_PORT}; +use crate::env::{CONTROL_SERVICE_ADDR, CONTROL_SERVICE_CA_CERT, CONTROL_SERVICE_PORT}; use crate::tls::CertStore; use arc_swap::ArcSwap; use nullnet_grpc_lib::NullnetGrpcInterface; @@ -7,6 +7,7 @@ use nullnet_grpc_lib::nullnet_grpc::{ }; use nullnet_liberror::{Error, ErrorHandler, Location, location}; use std::net::{IpAddr, SocketAddr}; +use std::path::Path; use std::sync::Arc; #[derive(Clone)] @@ -20,12 +21,12 @@ impl NullnetProxy { pub async fn new(certs: Arc>) -> Result { let host = CONTROL_SERVICE_ADDR.to_string(); let port = *CONTROL_SERVICE_PORT; + let ca_cert = CONTROL_SERVICE_CA_CERT + .as_deref() + .ok_or("'CONTROL_SERVICE_CA_CERT' environment variable must be set") + .handle_err(location!())?; - // TODO(grpc-tls): connecting with tls = false. Cert private keys are - // delivered over this channel via WatchCertificates, so enable TLS - // (tls = true) once the control service serves gRPC over TLS, so keys - // never travel in the clear. - let server = NullnetGrpcInterface::new(&host, port, false) + let server = NullnetGrpcInterface::new(&host, port, Path::new(ca_cert)) .await .handle_err(location!())?; diff --git a/members/nullnet-server/src/grpc_tls.rs b/members/nullnet-server/src/grpc_tls.rs new file mode 100644 index 00000000..e9f6776a --- /dev/null +++ b/members/nullnet-server/src/grpc_tls.rs @@ -0,0 +1,185 @@ +//! TLS identity for the gRPC control channel itself (distinct from the +//! customer-facing certs in `certs.rs`, which get streamed to proxies via +//! `WatchCertificates`). The server generates its own private CA on first +//! boot and signs its leaf with it; clients pin the stable CA root +//! (`ca-cert.pem`) via `CONTROL_SERVICE_CA_CERT` for full standard chain +//! validation, including hostname matching — only a leaf actually signed by +//! that CA, for the right host, is accepted. The CA is never regenerated +//! once created (pinned clients trust it, not the leaf), so future leaf +//! rotation needs no client-side changes. Because hostname matching +//! applies, the leaf's SAN must cover whatever host/IP clients use as +//! `CONTROL_SERVICE_ADDR` — set via `CONTROL_SERVICE_TLS_SAN` (see +//! `leaf_sans_from_env`). +use nullnet_liberror::{Error, ErrorHandler, Location, location}; +use rcgen::{ + BasicConstraints, CertificateParams, DistinguishedName, DnType, IsCa, Issuer, KeyPair, +}; +use std::path::{Path, PathBuf}; + +pub(crate) const GRPC_TLS_DIR: &str = "./grpc-tls"; +const CERT_FILE: &str = "cert.pem"; +const KEY_FILE: &str = "key.pem"; +const CA_CERT_FILE: &str = "ca-cert.pem"; +const CA_KEY_FILE: &str = "ca-key.pem"; + +/// Load the persisted cert/key, generating and persisting a CA-signed one +/// on first boot (see module docs). +pub(crate) async fn load_or_generate() -> Result<(String, String), Error> { + let dir = PathBuf::from(GRPC_TLS_DIR); + let cert_path = dir.join(CERT_FILE); + let key_path = dir.join(KEY_FILE); + + if let (Ok(cert_pem), Ok(key_pem)) = ( + tokio::fs::read_to_string(&cert_path).await, + tokio::fs::read_to_string(&key_path).await, + ) { + println!("Loaded gRPC control channel TLS certificate from '{GRPC_TLS_DIR}'"); + return Ok((cert_pem, key_pem)); + } + + tokio::fs::create_dir_all(&dir) + .await + .handle_err(location!())?; + + let (cert_pem, key_pem) = generate_ca_signed_leaf(&dir).await?; + + tokio::fs::write(&cert_path, &cert_pem) + .await + .handle_err(location!())?; + tokio::fs::write(&key_path, &key_pem) + .await + .handle_err(location!())?; + restrict_key_permissions(&key_path).await?; + + Ok((cert_pem, key_pem)) +} + +/// The CA's distinguished name must differ from the leaf's, or a validator +/// can't tell "signed by the CA" apart from "signed by itself" when the +/// issuer and subject fields are compared by name (both `openssl verify` +/// and rustls's WebPKI path-building do this) — an identical default name +/// on both was an earlier bug here, misreported as "self-signed certificate". +fn ca_distinguished_name() -> DistinguishedName { + let mut dn = DistinguishedName::new(); + dn.push(DnType::CommonName, "nullnet control-plane CA"); + dn +} + +fn leaf_distinguished_name() -> DistinguishedName { + let mut dn = DistinguishedName::new(); + dn.push(DnType::CommonName, "nullnet control channel"); + dn +} + +/// Comma-separated hostnames/IPs from `CONTROL_SERVICE_TLS_SAN`. Clients' +/// `PinnedCa` verifier does full standard validation, including hostname +/// matching — this must include whatever host/IP clients use as +/// `CONTROL_SERVICE_ADDR`, or their handshake fails with a hostname +/// mismatch, not a trust error. SANs are baked in when the leaf is signed; +/// changing this env var takes effect on the next leaf regeneration (i.e. +/// after deleting `cert.pem`/`key.pem` — the CA itself is untouched). +/// +/// Falls back to `CONTROL_SERVICE_ADDR` if unset — this server's own +/// `.env` often already sets that to the address clients are told to +/// connect to, which is exactly the SAN a self-consistent deployment needs +/// — and only then to `localhost`. +fn leaf_sans_from_env() -> Vec { + if let Ok(val) = std::env::var("CONTROL_SERVICE_TLS_SAN") { + let sans: Vec = val + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_string) + .collect(); + if !sans.is_empty() { + return sans; + } + } + + if let Ok(addr) = std::env::var("CONTROL_SERVICE_ADDR") { + let addr = addr.trim(); + if !addr.is_empty() { + println!( + "'CONTROL_SERVICE_TLS_SAN' environment variable not set, falling back to \ + 'CONTROL_SERVICE_ADDR' ('{addr}') for the leaf cert's SAN" + ); + return vec![addr.to_string()]; + } + } + + println!( + "Neither 'CONTROL_SERVICE_TLS_SAN' nor 'CONTROL_SERVICE_ADDR' environment variables are \ + set, defaulting the leaf cert's SAN to 'localhost' — set one of them to the host/IP \ + clients use to reach this server, or their handshake will fail hostname verification" + ); + vec!["localhost".to_string()] +} + +/// Load the persisted CA (generating it on first use), then sign a fresh +/// leaf cert with it. The CA cert itself is never regenerated once it +/// exists on disk — only its private key is reloaded, to sign a new leaf. +/// +/// Reconstructing `CertificateParams` here (rather than parsing the +/// existing `ca-cert.pem`, which rcgen has no public API for) is safe +/// because every field `Issuer::from_params` reads off it — distinguished +/// name, key identifier method, key usages — is set identically both here +/// and when the CA was first created, so they always match what's actually +/// embedded in the persisted CA cert. +async fn generate_ca_signed_leaf(dir: &Path) -> Result<(String, String), Error> { + let ca_cert_path = dir.join(CA_CERT_FILE); + let ca_key_path = dir.join(CA_KEY_FILE); + + let ca_key = if let Ok(ca_key_pem) = tokio::fs::read_to_string(&ca_key_path).await { + println!("Loaded gRPC control channel CA from '{GRPC_TLS_DIR}'"); + KeyPair::from_pem(&ca_key_pem).handle_err(location!())? + } else { + println!("Generating gRPC control channel private CA..."); + let mut ca_params = CertificateParams::new(vec![]).handle_err(location!())?; + ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + ca_params.distinguished_name = ca_distinguished_name(); + let ca_key = KeyPair::generate().handle_err(location!())?; + let ca_cert = ca_params.self_signed(&ca_key).handle_err(location!())?; + + tokio::fs::write(&ca_cert_path, ca_cert.pem()) + .await + .handle_err(location!())?; + tokio::fs::write(&ca_key_path, ca_key.serialize_pem()) + .await + .handle_err(location!())?; + restrict_key_permissions(&ca_key_path).await?; + println!( + "Generated gRPC control channel CA at '{}' — copy it to every client/proxy \ + host and point CONTROL_SERVICE_CA_CERT at it", + ca_cert_path.display() + ); + ca_key + }; + + println!("Signing gRPC control channel leaf certificate..."); + let mut ca_params = CertificateParams::new(vec![]).handle_err(location!())?; + ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + ca_params.distinguished_name = ca_distinguished_name(); + let issuer = Issuer::from_params(&ca_params, &ca_key); + + let mut leaf_params = CertificateParams::new(leaf_sans_from_env()).handle_err(location!())?; + leaf_params.distinguished_name = leaf_distinguished_name(); + let leaf_key = KeyPair::generate().handle_err(location!())?; + let leaf_cert = leaf_params + .signed_by(&leaf_key, &issuer) + .handle_err(location!())?; + + Ok((leaf_cert.pem(), leaf_key.serialize_pem())) +} + +#[cfg(unix)] +async fn restrict_key_permissions(key_path: &Path) -> Result<(), Error> { + use std::os::unix::fs::PermissionsExt; + tokio::fs::set_permissions(key_path, std::fs::Permissions::from_mode(0o600)) + .await + .handle_err(location!()) +} + +#[cfg(not(unix))] +async fn restrict_key_permissions(_key_path: &Path) -> Result<(), Error> { + Ok(()) +} diff --git a/members/nullnet-server/src/main.rs b/members/nullnet-server/src/main.rs index 2a2fabbb..87b1f584 100644 --- a/members/nullnet-server/src/main.rs +++ b/members/nullnet-server/src/main.rs @@ -6,6 +6,7 @@ mod env; mod events; mod geo; mod graphviz; +mod grpc_tls; mod http_server; mod net; mod net_id_pool; @@ -21,7 +22,7 @@ use nullnet_grpc_lib::nullnet_grpc::nullnet_grpc_server::NullnetGrpcServer; use nullnet_liberror::{Error, ErrorHandler, Location, location}; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::{panic, process}; -use tonic::transport::Server; +use tonic::transport::{Identity, Server, ServerTlsConfig}; const PORT: u16 = 50051; @@ -52,11 +53,14 @@ async fn main() -> Result<(), Error> { ); } - // TODO(grpc-tls): the gRPC server is plaintext, but WatchCertificates - // streams private keys. Enable TLS here (Server::builder().tls_config(..)), - // ideally mTLS, so keys never travel in clear; until then keep the control - // plane on a trusted network. Proxy side: NullnetGrpcInterface::new(.., true). - let mut server = Server::builder(); + // Server-only TLS: encrypts the channel (WatchCertificates streams + // customer private keys) but doesn't yet authenticate clients — mTLS is + // a separate, still-open follow-up for that. + let (cert_pem, key_pem) = grpc_tls::load_or_generate().await?; + let tls_config = ServerTlsConfig::new().identity(Identity::from_pem(cert_pem, key_pem)); + let mut server = Server::builder() + .tls_config(tls_config) + .handle_err(location!())?; let nullnet = init_nullnet().await?; let app_state = http_server::AppState { From 17dbfddd507c48e1843979e08aee227e9db5389e Mon Sep 17 00:00:00 2001 From: Anton Liashkevich Date: Tue, 21 Jul 2026 19:54:29 -0400 Subject: [PATCH 2/3] verify persisted leaf actually chains to the CA before reusing it A leftover leaf cert with no matching CA (e.g. from a pre-CA build of this feature) was being silently reused forever, so ca-cert.pem never got generated. Now the leaf is cryptographically verified against the persisted CA (issuer/subject match, signature, expiry) before reuse; any mismatch regenerates just the leaf, reusing the existing CA so client pins keep working. --- Cargo.lock | 1 + members/nullnet-server/Cargo.toml | 2 +- members/nullnet-server/src/grpc_tls.rs | 59 ++++++++++++++++++++++++-- 3 files changed, 58 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b2e07aeb..4c8b0406 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5644,6 +5644,7 @@ dependencies = [ "lazy_static", "nom", "oid-registry 0.7.1", + "ring", "rusticata-macros", "thiserror 1.0.69", "time", diff --git a/members/nullnet-server/Cargo.toml b/members/nullnet-server/Cargo.toml index 63e9472a..6f2473ae 100644 --- a/members/nullnet-server/Cargo.toml +++ b/members/nullnet-server/Cargo.toml @@ -32,7 +32,7 @@ rust-embed = { version = "8", features = ["axum"] } mime_guess = "2" aes-gcm = "0.10" base64 = "0.22" -x509-parser = "0.16" +x509-parser = { version = "0.16", features = ["verify"] } # ACME (Let's Encrypt) cert issuance via DNS-01, ported from ../routix anyhow = "1.0" instant-acme = { version = "0.8", features = ["hyper-rustls"] } diff --git a/members/nullnet-server/src/grpc_tls.rs b/members/nullnet-server/src/grpc_tls.rs index e9f6776a..62896a15 100644 --- a/members/nullnet-server/src/grpc_tls.rs +++ b/members/nullnet-server/src/grpc_tls.rs @@ -24,17 +24,35 @@ const CA_KEY_FILE: &str = "ca-key.pem"; /// Load the persisted cert/key, generating and persisting a CA-signed one /// on first boot (see module docs). +/// +/// Only reuses what's on disk if it actually verifies: the leaf must parse, +/// its issuer must match the CA's subject, and its signature must check out +/// against the CA's public key (see `verify_leaf_signed_by_ca`). Anything +/// short of that — a leaf without a matching CA (e.g. left over from a +/// pre-CA build of this feature), a mismatched pair, or plain corruption — +/// is treated as absent and regenerated, rather than silently served as the +/// server's TLS identity. pub(crate) async fn load_or_generate() -> Result<(String, String), Error> { let dir = PathBuf::from(GRPC_TLS_DIR); let cert_path = dir.join(CERT_FILE); let key_path = dir.join(KEY_FILE); + let ca_cert_path = dir.join(CA_CERT_FILE); - if let (Ok(cert_pem), Ok(key_pem)) = ( + if let (Ok(cert_pem), Ok(key_pem), Ok(ca_cert_pem)) = ( tokio::fs::read_to_string(&cert_path).await, tokio::fs::read_to_string(&key_path).await, + tokio::fs::read_to_string(&ca_cert_path).await, ) { - println!("Loaded gRPC control channel TLS certificate from '{GRPC_TLS_DIR}'"); - return Ok((cert_pem, key_pem)); + match verify_leaf_signed_by_ca(&cert_pem, &ca_cert_pem) { + Ok(()) => { + println!("Loaded gRPC control channel TLS certificate from '{GRPC_TLS_DIR}'"); + return Ok((cert_pem, key_pem)); + } + Err(e) => println!( + "Persisted leaf cert under '{GRPC_TLS_DIR}' failed verification against the \ + persisted CA ({e}) — regenerating the leaf" + ), + } } tokio::fs::create_dir_all(&dir) @@ -54,6 +72,41 @@ pub(crate) async fn load_or_generate() -> Result<(String, String), Error> { Ok((cert_pem, key_pem)) } +/// Verify `cert_pem` parses, is issued by `ca_cert_pem`'s subject, and its +/// signature checks out against the CA's public key — i.e. that this is +/// genuinely a leaf signed by this specific CA, not just two unrelated +/// files that happen to sit next to each other on disk. +fn verify_leaf_signed_by_ca(cert_pem: &str, ca_cert_pem: &str) -> Result<(), String> { + let (_, leaf_pem) = x509_parser::pem::parse_x509_pem(cert_pem.as_bytes()) + .map_err(|e| format!("failed to parse leaf cert: {e}"))?; + let leaf = leaf_pem + .parse_x509() + .map_err(|e| format!("failed to parse leaf cert: {e}"))?; + + let (_, ca_pem) = x509_parser::pem::parse_x509_pem(ca_cert_pem.as_bytes()) + .map_err(|e| format!("failed to parse CA cert: {e}"))?; + let ca = ca_pem + .parse_x509() + .map_err(|e| format!("failed to parse CA cert: {e}"))?; + + if leaf.tbs_certificate.issuer != ca.tbs_certificate.subject { + return Err("leaf cert's issuer does not match the CA's subject".to_string()); + } + leaf.verify_signature(Some(ca.public_key())) + .map_err(|e| format!("leaf cert signature does not verify against the CA: {e}"))?; + + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as i64; + let validity = leaf.validity(); + if now < validity.not_before.timestamp() || now > validity.not_after.timestamp() { + return Err("leaf cert is not currently valid (expired or not yet valid)".to_string()); + } + + Ok(()) +} + /// The CA's distinguished name must differ from the leaf's, or a validator /// can't tell "signed by the CA" apart from "signed by itself" when the /// issuer and subject fields are compared by name (both `openssl verify` From 9231fd427eb24a318b0f97dfc68a3170f9d0fd65 Mon Sep 17 00:00:00 2001 From: Anton Liashkevich Date: Thu, 23 Jul 2026 23:20:53 -0400 Subject: [PATCH 3/3] feat(grpc): default CONTROL_SERVICE_CA_CERT to ./ca-cert.pem MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Client/proxy no longer hard-require the env var — if unset they fall back to ca-cert.pem in the working directory and only fail once they actually try to read it (missing file or invalid cert), instead of refusing to start over an unset variable that already has a sane default. --- README.md | 24 +++++++++++++--------- members/nullnet-client/src/env.rs | 11 ++++++---- members/nullnet-client/src/main.rs | 12 ++++------- members/nullnet-grpc-lib/src/lib.rs | 4 +++- members/nullnet-proxy/src/env.rs | 11 ++++++---- members/nullnet-proxy/src/nullnet_proxy.rs | 12 ++++------- 6 files changed, 39 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index d095bd12..7ca18386 100644 --- a/README.md +++ b/README.md @@ -81,10 +81,10 @@ The repository should be cloned under `/root` so the provided `setup-*.sh` scrip authenticated by a private CA. On first boot the server generates its own CA (`members/nullnet-server/grpc-tls/ca-cert.pem` + `ca-key.pem`, created once and never regenerated) and signs its leaf cert with it. Copy `ca-cert.pem` to every client/proxy host and - set `CONTROL_SERVICE_CA_CERT` there (see below) — it's **required**: clients pin the channel to - that CA and do full standard chain validation, so only a leaf actually signed by it is accepted. - Because clients trust the stable CA root rather than the leaf, rotating the leaf later needs no - client-side changes. Client authentication (mTLS) remains a further follow-up. + set `CONTROL_SERVICE_CA_CERT` there (see below) if it isn't at the default path — clients pin the + channel to that CA and do full standard chain validation, so only a leaf actually signed by it is + accepted. Because clients trust the stable CA root rather than the leaf, rotating the leaf later + needs no client-side changes. Client authentication (mTLS) remains a further follow-up. Validation includes hostname matching, so the leaf's SAN must cover whatever host/IP clients use as `CONTROL_SERVICE_ADDR`. It's derived in this order: `CONTROL_SERVICE_TLS_SAN` @@ -197,10 +197,12 @@ The repository should be cloned under `/root` so the provided `setup-*.sh` scrip ``` CONTROL_SERVICE_ADDR=192.168.1.100 CONTROL_SERVICE_PORT=50051 - CONTROL_SERVICE_CA_CERT=./ca-cert.pem + CONTROL_SERVICE_CA_CERT=./ca-cert.pem # optional, defaults to ./ca-cert.pem ``` - `CONTROL_SERVICE_CA_CERT` is **required** — point it at a copy of the server's own - `grpc-tls/ca-cert.pem` (see the server section above) to pin and authenticate the control channel. + `CONTROL_SERVICE_CA_CERT` points at a copy of the server's own `grpc-tls/ca-cert.pem` (see the + server section above), used to pin and authenticate the control channel. If unset it defaults to + `ca-cert.pem` in the working directory; either way, startup fails if the file is missing or isn't + a valid cert. - run the project as a daemon (from the repo root) ``` @@ -223,10 +225,12 @@ The repository should be cloned under `/root` so the provided `setup-*.sh` scrip ``` CONTROL_SERVICE_ADDR=192.168.1.100 CONTROL_SERVICE_PORT=50051 - CONTROL_SERVICE_CA_CERT=./ca-cert.pem + CONTROL_SERVICE_CA_CERT=./ca-cert.pem # optional, defaults to ./ca-cert.pem ``` - `CONTROL_SERVICE_CA_CERT` is **required** — point it at a copy of the server's own - `grpc-tls/ca-cert.pem` (see the server section above) to pin and authenticate the control channel. + `CONTROL_SERVICE_CA_CERT` points at a copy of the server's own `grpc-tls/ca-cert.pem` (see the + server section above), used to pin and authenticate the control channel. If unset it defaults to + `ca-cert.pem` in the working directory; either way, startup fails if the file is missing or isn't + a valid cert. > **⚠️ The client attaches a default-deny eBPF firewall to the uplink NIC on startup.** It permits > only the nullnet control plane (gRPC to the server), data plane (VXLAN to peers), established diff --git a/members/nullnet-client/src/env.rs b/members/nullnet-client/src/env.rs index 16a981f6..e0305413 100644 --- a/members/nullnet-client/src/env.rs +++ b/members/nullnet-client/src/env.rs @@ -15,7 +15,10 @@ pub static CONTROL_SERVICE_PORT: std::sync::LazyLock = std::sync::LazyLock: }); /// Path to the control server's private CA root (its `grpc-tls/ca-cert.pem` -/// — generated automatically on the server, copy it here). Required: pins -/// the control channel to that CA for full standard chain validation. -pub static CONTROL_SERVICE_CA_CERT: std::sync::LazyLock> = - std::sync::LazyLock::new(|| std::env::var("CONTROL_SERVICE_CA_CERT").ok()); +/// — generated automatically on the server, copy it here) — pins the control +/// channel to that CA for full standard chain validation. Defaults to +/// `ca-cert.pem` (relative to the working directory) if unset; the connection +/// fails at startup if the file is missing or not a valid cert. +pub static CONTROL_SERVICE_CA_CERT: std::sync::LazyLock = std::sync::LazyLock::new(|| { + std::env::var("CONTROL_SERVICE_CA_CERT").unwrap_or_else(|_| "ca-cert.pem".to_string()) +}); diff --git a/members/nullnet-client/src/main.rs b/members/nullnet-client/src/main.rs index 855e9b84..05af4eea 100644 --- a/members/nullnet-client/src/main.rs +++ b/members/nullnet-client/src/main.rs @@ -281,14 +281,10 @@ fn resolve_server_ip() -> Option { async fn grpc_init() -> Result { let host = CONTROL_SERVICE_ADDR.to_string(); let port = *CONTROL_SERVICE_PORT; - let ca_cert = CONTROL_SERVICE_CA_CERT - .as_deref() - .ok_or("'CONTROL_SERVICE_CA_CERT' environment variable must be set") - .handle_err(location!())?; - - let server = NullnetGrpcInterface::new(&host, port, Path::new(ca_cert)) - .await - .handle_err(location!())?; + let server = + NullnetGrpcInterface::new(&host, port, Path::new(CONTROL_SERVICE_CA_CERT.as_str())) + .await + .handle_err(location!())?; Ok(server) } diff --git a/members/nullnet-grpc-lib/src/lib.rs b/members/nullnet-grpc-lib/src/lib.rs index 8783d367..a0f0d0bb 100644 --- a/members/nullnet-grpc-lib/src/lib.rs +++ b/members/nullnet-grpc-lib/src/lib.rs @@ -27,7 +27,9 @@ impl NullnetGrpcInterface { /// hostname matching (see `control_tls_verifier::PinnedCa`). #[allow(clippy::missing_errors_doc)] pub async fn new(host: &str, port: u16, ca_cert: &Path) -> Result { - let ca_cert_pem = tokio::fs::read(ca_cert).await.map_err(|e| e.to_string())?; + let ca_cert_pem = tokio::fs::read(ca_cert) + .await + .map_err(|e| format!("failed to read CA cert at {}: {e}", ca_cert.display()))?; let verifier = PinnedCa::verifier(&ca_cert_pem)?; // Keepalive so a dead/unreachable server breaks open streams within diff --git a/members/nullnet-proxy/src/env.rs b/members/nullnet-proxy/src/env.rs index 16a981f6..e0305413 100644 --- a/members/nullnet-proxy/src/env.rs +++ b/members/nullnet-proxy/src/env.rs @@ -15,7 +15,10 @@ pub static CONTROL_SERVICE_PORT: std::sync::LazyLock = std::sync::LazyLock: }); /// Path to the control server's private CA root (its `grpc-tls/ca-cert.pem` -/// — generated automatically on the server, copy it here). Required: pins -/// the control channel to that CA for full standard chain validation. -pub static CONTROL_SERVICE_CA_CERT: std::sync::LazyLock> = - std::sync::LazyLock::new(|| std::env::var("CONTROL_SERVICE_CA_CERT").ok()); +/// — generated automatically on the server, copy it here) — pins the control +/// channel to that CA for full standard chain validation. Defaults to +/// `ca-cert.pem` (relative to the working directory) if unset; the connection +/// fails at startup if the file is missing or not a valid cert. +pub static CONTROL_SERVICE_CA_CERT: std::sync::LazyLock = std::sync::LazyLock::new(|| { + std::env::var("CONTROL_SERVICE_CA_CERT").unwrap_or_else(|_| "ca-cert.pem".to_string()) +}); diff --git a/members/nullnet-proxy/src/nullnet_proxy.rs b/members/nullnet-proxy/src/nullnet_proxy.rs index e9e3af2e..a59711e5 100644 --- a/members/nullnet-proxy/src/nullnet_proxy.rs +++ b/members/nullnet-proxy/src/nullnet_proxy.rs @@ -21,14 +21,10 @@ impl NullnetProxy { pub async fn new(certs: Arc>) -> Result { let host = CONTROL_SERVICE_ADDR.to_string(); let port = *CONTROL_SERVICE_PORT; - let ca_cert = CONTROL_SERVICE_CA_CERT - .as_deref() - .ok_or("'CONTROL_SERVICE_CA_CERT' environment variable must be set") - .handle_err(location!())?; - - let server = NullnetGrpcInterface::new(&host, port, Path::new(ca_cert)) - .await - .handle_err(location!())?; + let server = + NullnetGrpcInterface::new(&host, port, Path::new(CONTROL_SERVICE_CA_CERT.as_str())) + .await + .handle_err(location!())?; Ok(Self { server,