Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .workhorse/specs/public-server/restore-replicas.md
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,7 @@ Reports are retained indefinitely as an audit trail.

Canopy derives each restore's duration from the interval between its first credential issuance and its report.
A restore for which credentials were issued but no report has arrived is shown as in progress while the credentials remain valid, otherwise as a restore whose outcome is unknown; this surfaces in-flight and terminated-without-report restores in the operator view, including those under intents that produce no health report.
The derivation covers consumers only: a device belonging to one of the group's servers takes restore credentials for its own purposes (a clone refresh, an operator's manual restore) and never reports, so its issuances derive no restore activity.

## Pre-upgrade migration testing

Expand Down
13 changes: 12 additions & 1 deletion crates/private-server/src/fns/migration_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -82,13 +82,24 @@ pub async fn attempt_state(

let reports =
database::restore::BackupRestoreCheck::list_recent_for_group(conn, group_id, 50).await?;
// Member-server devices restore for their own purposes (clone refreshes,
// manual restores) and never report; only consumer issuances speak for the
// pipeline. Same filter as the restore-activity view.
let member_devices: Vec<Uuid> = database::servers::Server::list_live_in_group(conn, group_id)
.await?
.into_iter()
.filter_map(|s| s.device_id)
.collect();
let since =
crate::run_pairing::issuance_since(now, reports.iter().map(|c| c.reported_at).min());
let issuances: Vec<_> =
BackupCredentialIssuance::list_for_group_since(conn, group_id, since, 200)
.await?
.into_iter()
.filter(|i| i.purpose == commons_types::backup::BackupPurpose::Restore)
.filter(|i| {
i.purpose == commons_types::backup::BackupPurpose::Restore
&& !member_devices.contains(&i.device_id)
})
.collect();

let report_refs: Vec<crate::run_pairing::ReportRef> = reports
Expand Down
69 changes: 69 additions & 0 deletions crates/private-server/tests/it/upgrade_plans.rs
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,75 @@ async fn an_attempt_in_flight_shows_beside_the_verdict() {
.await;
}

/// A member server taking restore credentials (a clone refresh, a manual
/// restore) never reports, so its expired issuances must not read as a test run
/// that ended without reporting.
#[tokio::test(flavor = "multi_thread")]
async fn a_member_servers_own_restore_is_not_an_attempt() {
commons_tests::server::run(async |mut conn, _, private| {
conn.batch_execute(
"INSERT INTO versions (id, major, minor, patch, changelog, status) VALUES
('cccccccc-0000-0000-0000-0000000000f1', 2, 61, 0, 'x', 'published');
INSERT INTO server_groups (id, name, effective_version) VALUES
('cccccccc-0000-0000-0000-000000000001', 'kamaka', '2.60.0');
INSERT INTO devices (id, role) VALUES
('cccccccc-0000-0000-0000-0000000000d0', 'backup-restore'),
('cccccccc-0000-0000-0000-0000000000d1', 'server');
INSERT INTO servers (id, name, host, kind, group_id, device_id) VALUES
('cccccccc-0000-0000-0000-0000000000a0', 'clone',
'https://clone.example.com', 'central',
'cccccccc-0000-0000-0000-000000000001',
'cccccccc-0000-0000-0000-0000000000d1');
INSERT INTO upgrade_plans (group_id, target_version_id) VALUES
('cccccccc-0000-0000-0000-000000000001',
'cccccccc-0000-0000-0000-0000000000f1');
INSERT INTO backup_credential_issuances
(device_id, group_id, type, purpose, run_id, issued_at, expires_at,
sts_assumed_role, bucket, prefix)
VALUES ('cccccccc-0000-0000-0000-0000000000d1',
'cccccccc-0000-0000-0000-000000000001', 'tamanu-postgres', 'restore',
'cccccccc-0000-0000-0000-0000000000e1',
NOW() - INTERVAL '3 hours', NOW() - INTERVAL '2 hours',
'arn:aws:iam::1:role/r', 'b', '')",
)
.await
.unwrap();

let fleet: Vec<Value> = private
.post("/api/upgrade_plans/fleet")
.json(&json!({}))
.await
.json();
let row = fleet.iter().find(|r| r["group_id"] == GROUP).unwrap();
assert!(
row["attempt"].is_null(),
"a member server's own restore is not the pipeline"
);

// The same expired-unreported issuance from the consumer is the signal.
conn.batch_execute(
"INSERT INTO backup_credential_issuances
(device_id, group_id, type, purpose, issued_at, expires_at,
sts_assumed_role, bucket, prefix)
VALUES ('cccccccc-0000-0000-0000-0000000000d0',
'cccccccc-0000-0000-0000-000000000001', 'tamanu-postgres', 'restore',
NOW() - INTERVAL '3 hours', NOW() - INTERVAL '2 hours',
'arn:aws:iam::1:role/r', 'b', '')",
)
.await
.unwrap();

let fleet: Vec<Value> = private
.post("/api/upgrade_plans/fleet")
.json(&json!({}))
.await
.json();
let row = fleet.iter().find(|r| r["group_id"] == GROUP).unwrap();
assert_eq!(row["attempt"], "ended_without_report");
})
.await;
}

#[tokio::test(flavor = "multi_thread")]
async fn amend_changes_the_date_and_note_without_replacing_the_plan() {
commons_tests::server::run(async |mut conn, _, private| {
Expand Down