diff --git a/.workhorse/specs/public-server/restore-replicas.md b/.workhorse/specs/public-server/restore-replicas.md index c9e363d0..b092eddc 100644 --- a/.workhorse/specs/public-server/restore-replicas.md +++ b/.workhorse/specs/public-server/restore-replicas.md @@ -209,6 +209,7 @@ Reports are retained indefinitely as an audit trail. Canopy derives each restore's duration from the interval between its first credential issuance and its report. A restore for which credentials were issued but no report has arrived is shown as in progress while the credentials remain valid, otherwise as a restore whose outcome is unknown; this surfaces in-flight and terminated-without-report restores in the operator view, including those under intents that produce no health report. +The derivation covers consumers only: a device belonging to one of the group's servers takes restore credentials for its own purposes (a clone refresh, an operator's manual restore) and never reports, so its issuances derive no restore activity. ## Pre-upgrade migration testing diff --git a/crates/private-server/src/fns/migration_tests.rs b/crates/private-server/src/fns/migration_tests.rs index 52cdacdd..24c1764d 100644 --- a/crates/private-server/src/fns/migration_tests.rs +++ b/crates/private-server/src/fns/migration_tests.rs @@ -82,13 +82,24 @@ pub async fn attempt_state( let reports = database::restore::BackupRestoreCheck::list_recent_for_group(conn, group_id, 50).await?; + // Member-server devices restore for their own purposes (clone refreshes, + // manual restores) and never report; only consumer issuances speak for the + // pipeline. Same filter as the restore-activity view. + let member_devices: Vec = database::servers::Server::list_live_in_group(conn, group_id) + .await? + .into_iter() + .filter_map(|s| s.device_id) + .collect(); let since = crate::run_pairing::issuance_since(now, reports.iter().map(|c| c.reported_at).min()); let issuances: Vec<_> = BackupCredentialIssuance::list_for_group_since(conn, group_id, since, 200) .await? .into_iter() - .filter(|i| i.purpose == commons_types::backup::BackupPurpose::Restore) + .filter(|i| { + i.purpose == commons_types::backup::BackupPurpose::Restore + && !member_devices.contains(&i.device_id) + }) .collect(); let report_refs: Vec = reports diff --git a/crates/private-server/tests/it/upgrade_plans.rs b/crates/private-server/tests/it/upgrade_plans.rs index 043a4b30..0ebd3421 100644 --- a/crates/private-server/tests/it/upgrade_plans.rs +++ b/crates/private-server/tests/it/upgrade_plans.rs @@ -146,6 +146,75 @@ async fn an_attempt_in_flight_shows_beside_the_verdict() { .await; } +/// A member server taking restore credentials (a clone refresh, a manual +/// restore) never reports, so its expired issuances must not read as a test run +/// that ended without reporting. +#[tokio::test(flavor = "multi_thread")] +async fn a_member_servers_own_restore_is_not_an_attempt() { + commons_tests::server::run(async |mut conn, _, private| { + conn.batch_execute( + "INSERT INTO versions (id, major, minor, patch, changelog, status) VALUES + ('cccccccc-0000-0000-0000-0000000000f1', 2, 61, 0, 'x', 'published'); + INSERT INTO server_groups (id, name, effective_version) VALUES + ('cccccccc-0000-0000-0000-000000000001', 'kamaka', '2.60.0'); + INSERT INTO devices (id, role) VALUES + ('cccccccc-0000-0000-0000-0000000000d0', 'backup-restore'), + ('cccccccc-0000-0000-0000-0000000000d1', 'server'); + INSERT INTO servers (id, name, host, kind, group_id, device_id) VALUES + ('cccccccc-0000-0000-0000-0000000000a0', 'clone', + 'https://clone.example.com', 'central', + 'cccccccc-0000-0000-0000-000000000001', + 'cccccccc-0000-0000-0000-0000000000d1'); + INSERT INTO upgrade_plans (group_id, target_version_id) VALUES + ('cccccccc-0000-0000-0000-000000000001', + 'cccccccc-0000-0000-0000-0000000000f1'); + INSERT INTO backup_credential_issuances + (device_id, group_id, type, purpose, run_id, issued_at, expires_at, + sts_assumed_role, bucket, prefix) + VALUES ('cccccccc-0000-0000-0000-0000000000d1', + 'cccccccc-0000-0000-0000-000000000001', 'tamanu-postgres', 'restore', + 'cccccccc-0000-0000-0000-0000000000e1', + NOW() - INTERVAL '3 hours', NOW() - INTERVAL '2 hours', + 'arn:aws:iam::1:role/r', 'b', '')", + ) + .await + .unwrap(); + + let fleet: Vec = private + .post("/api/upgrade_plans/fleet") + .json(&json!({})) + .await + .json(); + let row = fleet.iter().find(|r| r["group_id"] == GROUP).unwrap(); + assert!( + row["attempt"].is_null(), + "a member server's own restore is not the pipeline" + ); + + // The same expired-unreported issuance from the consumer is the signal. + conn.batch_execute( + "INSERT INTO backup_credential_issuances + (device_id, group_id, type, purpose, issued_at, expires_at, + sts_assumed_role, bucket, prefix) + VALUES ('cccccccc-0000-0000-0000-0000000000d0', + 'cccccccc-0000-0000-0000-000000000001', 'tamanu-postgres', 'restore', + NOW() - INTERVAL '3 hours', NOW() - INTERVAL '2 hours', + 'arn:aws:iam::1:role/r', 'b', '')", + ) + .await + .unwrap(); + + let fleet: Vec = private + .post("/api/upgrade_plans/fleet") + .json(&json!({})) + .await + .json(); + let row = fleet.iter().find(|r| r["group_id"] == GROUP).unwrap(); + assert_eq!(row["attempt"], "ended_without_report"); + }) + .await; +} + #[tokio::test(flavor = "multi_thread")] async fn amend_changes_the_date_and_note_without_replacing_the_plan() { commons_tests::server::run(async |mut conn, _, private| {