feat: expiry warnings, per-site off-site backups, per-site fail2ban, verified node→master TLS

bin/hyperion-agent/src/config.rs

@@ -108,15 +108,25 @@ pub struct EnrollmentSection {
     /// Path where the assigned node_id is persisted after first enrollment.
     /// Defaults to /etc/hyperion/node-id.json.
     pub state_file: Option<PathBuf>,
-    /// When `true`, the node verifies the master's TLS cert against
-    /// the system CA bundle. Defaults to `false` because install-
-    /// master.sh ships a self-signed cert (no DNS at install time
-    /// → no LE) and the node has no trust anchor to bootstrap.
-    /// The bearer token + per-node secret are the auth; TLS here
-    /// is encryption-in-transit. Set `true` once the master serves
-    /// a real LE cert.
+    /// TLS verification for the node→master leg — enrollment, and every
+    /// heartbeat, which carries this node's plaintext secret.
+    ///
+    /// Three states, and the ABSENT one is the point:
+    /// - key absent ⇒ **auto**: verify whenever `master_url` is
+    ///   `https://` with a DNS hostname, because that is the shape a
+    ///   CA-issued master certificate has. The master is normally the
+    ///   side that HAS a real certificate; the worker is the side that
+    ///   cannot, which is why this direction can be verified today and
+    ///   the master→worker one still leans on cert pinning.
+    /// - `true` ⇒ always verify, even for an IP-literal master URL.
+    /// - `false` ⇒ the documented escape hatch for a master serving a
+    ///   self-signed certificate. The invite token and the per-node
+    ///   secret then cross a channel an on-path attacker can read.
+    ///
+    /// A failed verification is never retried unverified: the agent
+    /// aborts and names the fix. See `enroll::decide_verify_tls`.
     #[serde(default)]
-    pub verify_tls: bool,
+    pub verify_tls: Option<bool>,
 }
 
 #[derive(Debug, Clone, Deserialize)]
@@ -290,6 +300,24 @@ mod tests {
         assert_eq!(cfg.acme.contact_email, "admin@example.com");
     }
 
+    /// The tri-state is load-bearing: an agent.toml that never mentions
+    /// `verify_tls` must be distinguishable from one that deliberately
+    /// switched verification OFF, because those two get opposite
+    /// treatment (auto-decide vs. honour the escape hatch).
+    #[test]
+    fn verify_tls_absent_is_not_the_same_as_false() {
+        let absent: Config =
+            toml::from_str("[enrollment]\nmaster_url = \"https://m.example.cz\"").expect("parse");
+        assert_eq!(absent.enrollment.verify_tls, None);
+        // Not even a whole missing section flips it to a decision.
+        assert_eq!(Config::default().enrollment.verify_tls, None);
+
+        let off: Config = toml::from_str("[enrollment]\nverify_tls = false").expect("parse");
+        assert_eq!(off.enrollment.verify_tls, Some(false));
+        let on: Config = toml::from_str("[enrollment]\nverify_tls = true").expect("parse");
+        assert_eq!(on.enrollment.verify_tls, Some(true));
+    }
+
     #[test]
     fn partial_toml_overrides_default() {
         let toml = r#"

bin/hyperion-web/src/dispatcher.rs

@@ -47,6 +47,14 @@ pub enum DispatchError {
     /// it never carries response content.
     #[error("node {node_id} response failed authentication: {reason}")]
     ResponseAuthFailed { node_id: String, reason: String },
+    /// Worker TLS certificate pinning is ENFORCED and the master holds
+    /// no pin for this node, so the connection would be made with no
+    /// certificate check at all. Same family as `ResponseAuthFailed`
+    /// and deliberately not `NodeUnreachable`: nothing is wrong with
+    /// the node's reachability, the master simply refuses to open an
+    /// unauthenticated channel while the operator has said not to.
+    #[error("node {node_id} has no TLS certificate pin on file while pinning is enforced")]
+    CertPinMissing { node_id: String },
     #[error("target node {0} is not enrolled")]
     UnknownNode(String),
     #[error("target node {0} has no public_ip on record — cannot reach")]
@@ -203,16 +211,15 @@ async fn dispatch_remote(
     // trip") — but "this node has published nothing" is exactly the state
     // an on-path attacker engineers by stripping resp_pubkey from the
     // heartbeat, and skipping the read handed that node a hard-coded
-    // enforce=false. `check_response_auth` needs the real toggle value to
-    // refuse it. Cert pinning loses nothing either way: with no reported
-    // pin there is nothing to pass to --pinnedpubkey regardless.
+    // enforce=false. BOTH checks below need the real toggle value to
+    // refuse a node in that state — an absent TLS pin is engineered the
+    // same way, by stripping tls_spki_pin from the heartbeat.
     let enforce = cluster_enforcement(state).await;
-    // Block C enforce phase: when the cluster toggle is on AND this node
-    // has a heartbeat-reported pin, pin it for real (curl --pinnedpubkey).
-    let pinned_pubkey = match &route.reported_pin {
-        Some(pin) if enforce.cert_pinning => Some(pin.clone()),
-        _ => None,
-    };
+    // Block C enforce phase. Refuses BEFORE dialling when there is no pin
+    // to enforce with — see check_cert_pinning for why "no pin" cannot be
+    // treated as "nothing to enforce".
+    let pinned_pubkey =
+        check_cert_pinning(node_id, route.reported_pin.as_deref(), enforce.cert_pinning)?;
     let opts = RemoteCallOpts {
         timeout_secs: timeout_for_request(&req),
         pinned_pubkey,
@@ -334,12 +341,16 @@ pub async fn fan_out_reporting(
                     // worker". Log it at ERROR with the security wording
                     // so it can't be lost among routine exclusions in
                     // `journalctl -u hyperion-web -g fan_out`.
-                    if matches!(e, DispatchError::ResponseAuthFailed { .. }) {
+                    if matches!(
+                        e,
+                        DispatchError::ResponseAuthFailed { .. }
+                            | DispatchError::CertPinMissing { .. }
+                    ) {
                         tracing::error!(
                             node = %nid,
                             error = %e,
-                            "SECURITY: fan_out excluded a node whose response failed \
-                             authentication — this page is rendering an INCOMPLETE \
+                            "SECURITY: fan_out excluded a node the master could not \
+                             authenticate — this page is rendering an INCOMPLETE \
                              aggregate, not an empty one"
                         );
                     } else {
@@ -449,6 +460,53 @@ async fn cluster_enforcement(state: &SharedState) -> Enforcement {
     }
 }
 
+/// Decide the `--pinnedpubkey` value for this dispatch, or refuse it.
+///
+/// The pin the master holds is trust-on-first-use and **write-once**:
+/// `nodes.tls_spki_pin` is only ever FILLED (`COALESCE(tls_spki_pin, ?)`
+/// in `touch_last_seen`), a heartbeat presenting a DIFFERENT pin is
+/// refused and warned about (`tofu_report`), and clearing it is an
+/// explicit operator action (`node_reset_crypto`, re-enrollment). So an
+/// attacker cannot silently re-aim a pin that has landed.
+///
+/// What they can still do is stop one from ever landing: the heartbeat
+/// travels over `curl -k`, so stripping `tls_spki_pin` from it leaves
+/// the column NULL forever. Treating NULL as "nothing to enforce" would
+/// hand the attacker the choice of WHICH nodes the toggle protects, and
+/// the connection would then be made with `-k` and no pin — precisely
+/// the state the toggle exists to end. So under enforcement, no pin on
+/// file is a refusal, exactly like no response-signing key on file is in
+/// [`check_response_auth`].
+///
+/// With enforcement OFF this returns `Ok(None)` for every input and
+/// nothing is ever refused: the warn-only observation in
+/// [`warn_on_pin_mismatch`] is the whole behaviour, unchanged.
+fn check_cert_pinning(
+    node_id: &str,
+    reported_pin: Option<&str>,
+    enforce: bool,
+) -> Result<Option<String>, DispatchError> {
+    if !enforce {
+        return Ok(None);
+    }
+    match reported_pin {
+        Some(pin) => Ok(Some(pin.to_string())),
+        None => {
+            tracing::error!(
+                node = node_id,
+                "SECURITY: worker TLS certificate pinning is enforced but this node has no \
+                 pin on file, so the RPC would run over an unverified connection with nothing \
+                 to check the cert against. Refused. Restart the node's agent and wait one \
+                 heartbeat for it to report its pin (the 🔒 chip on Nodes), or turn off \
+                 Enforce worker TLS certificate pinning in Settings → Cluster."
+            );
+            Err(DispatchError::CertPinMissing {
+                node_id: node_id.to_string(),
+            })
+        }
+    }
+}
+
 /// The four-way response-authentication matrix, evaluated on every
 /// remote dispatch. `resp_pubkey` is the key the node published over
 /// its authenticated heartbeat; `out.resp_sig` is what arrived on
@@ -653,6 +711,17 @@ impl From<DispatchError> for crate::error::AppError {
                  connectivity problem: check `journalctl -u hyperion-agent` on the node \
                  and whether anything is intercepting master→worker traffic."
             )),
+            // Also NOT NodeUnreachable: the node is fine, the master
+            // declined to talk to it unverified. The two fixes are named
+            // in the order an operator should try them — re-reporting the
+            // pin keeps the protection, switching the toggle off drops it.
+            DispatchError::CertPinMissing { node_id } => AppError::Rpc(format!(
+                "node {node_id} has not reported a TLS certificate pin, and Enforce worker TLS \
+                 certificate pinning is on in Settings → Cluster — so the master refused to \
+                 dispatch over a connection it cannot check. Restart hyperion-agent on that node \
+                 and wait one heartbeat (about 30 s) for the 🔒 chip to appear on Nodes, or turn \
+                 the setting off."
+            )),
             DispatchError::UnknownNode(n) => AppError::BadRequest(format!(
                 "node {n} is not enrolled — pick a different target"
             )),
@@ -792,6 +861,67 @@ mod tests {
         assert!(is_auth_failure(&err), "wrong variant: {err}");
     }
 
+    const PIN: &str = "/4IrPU/vEdcxQgcB9m3gD/9oaQ9/8WmdvXZIDD+ZVxg=";
+
+    /// Nothing changes until the operator flips the toggle — including
+    /// for a node that has never reported a pin. This is the arm that
+    /// keeps a mixed cluster working, so it is asserted first.
+    #[test]
+    fn cert_pinning_is_inert_until_the_operator_enforces_it() {
+        assert_eq!(check_cert_pinning(NODE, Some(PIN), false).unwrap(), None);
+        assert_eq!(check_cert_pinning(NODE, None, false).unwrap(), None);
+    }
+
+    /// Enforcing pins the value on file, verbatim — it is what curl gets
+    /// after `sha256//`, so any rewriting here would break every call.
+    #[test]
+    fn enforced_pinning_pins_exactly_the_pin_on_file() {
+        assert_eq!(
+            check_cert_pinning(NODE, Some(PIN), true)
+                .expect("a node with a pin is dispatched to")
+                .as_deref(),
+            Some(PIN)
+        );
+    }
+
+    /// The write-once rule's consequence. Because the master only ever
+    /// FILLS `nodes.tls_spki_pin` and refuses a changed one, an attacker
+    /// cannot re-aim a pin that landed — the only lever left is keeping
+    /// one from ever landing, by stripping `tls_spki_pin` from the
+    /// node's (unverified) heartbeats. Enforcement therefore has to read
+    /// "no pin on file" as a refusal; reading it as "nothing to check"
+    /// would let the attacker choose who the toggle protects.
+    #[test]
+    fn enforced_pinning_refuses_a_node_whose_pin_never_landed() {
+        let err = check_cert_pinning(NODE, None, true)
+            .expect_err("a pinless node must not be dispatched to under enforcement");
+        assert!(
+            matches!(err, DispatchError::CertPinMissing { .. }),
+            "wrong variant: {err}"
+        );
+        assert!(err.to_string().contains("no TLS certificate pin"), "{err}");
+    }
+
+    /// ...and the refusal must not read as downtime either: an operator
+    /// who restarts a healthy agent looking for a network fault is an
+    /// operator who never finds the toggle that caused this.
+    #[test]
+    fn missing_pin_does_not_render_as_node_unreachable() {
+        let app: crate::error::AppError = DispatchError::CertPinMissing {
+            node_id: NODE.to_string(),
+        }
+        .into();
+        assert!(
+            !matches!(app, crate::error::AppError::NodeUnreachable { .. }),
+            "an enforcement refusal must not present as downtime"
+        );
+        let text = app.to_string();
+        assert!(
+            text.contains("Settings"),
+            "names where to turn it off: {text}"
+        );
+    }
+
     /// A forged response must never reach the operator as "node
     /// unreachable" — that reads as downtime and invites a retry.
     #[test]