From 511a6f87ff4a257de5814dafeda2c3e3c7197c67 Mon Sep 17 00:00:00 2001 From: Josh Rotenberg Date: Tue, 11 Aug 2026 14:22:56 -0700 Subject: [PATCH 1/2] chore: start scheduled live validation From 7347fa13c4a53af0b2452fc22b9f7b9abc3f0305 Mon Sep 17 00:00:00 2001 From: Josh Rotenberg Date: Tue, 11 Aug 2026 14:32:50 -0700 Subject: [PATCH 2/2] ci(live): schedule safe contract validation --- .github/workflows/live-contract.yml | 135 +++++++++++++ tests/compliance.rs | 302 ++++++++++++++++++---------- tests/fixtures/README.md | 39 ++++ tests/live_integration.rs | 19 +- 4 files changed, 386 insertions(+), 109 deletions(-) create mode 100644 .github/workflows/live-contract.yml diff --git a/.github/workflows/live-contract.yml b/.github/workflows/live-contract.yml new file mode 100644 index 0000000..24fcacc --- /dev/null +++ b/.github/workflows/live-contract.yml @@ -0,0 +1,135 @@ +name: Redis Cloud Live Contract Validation + +on: + schedule: + - cron: "37 14 * * 3" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: redis-cloud-live-contract-validation + cancel-in-progress: false + +jobs: + validate: + name: Validate safe live contracts + runs-on: ubuntu-latest + timeout-minutes: 30 + environment: live-contract-tests + env: + REDIS_CLOUD_API_ACCOUNT_KEY: ${{ secrets.REDIS_CLOUD_API_ACCOUNT_KEY }} + REDIS_CLOUD_API_USER_KEY: ${{ secrets.REDIS_CLOUD_API_USER_KEY }} + REDIS_CLOUD_TEST_PRO_SUB_ID: ${{ secrets.REDIS_CLOUD_TEST_PRO_SUB_ID }} + REDIS_CLOUD_TEST_PRO_DB_ID: ${{ secrets.REDIS_CLOUD_TEST_PRO_DB_ID }} + REDIS_CLOUD_TEST_ESSENTIALS_SUB_ID: ${{ secrets.REDIS_CLOUD_TEST_ESSENTIALS_SUB_ID }} + REDIS_CLOUD_TEST_ESSENTIALS_DB_ID: ${{ secrets.REDIS_CLOUD_TEST_ESSENTIALS_DB_ID }} + steps: + - uses: actions/checkout@v4 + + - name: Verify dedicated test configuration + shell: bash + run: | + missing=0 + for name in \ + REDIS_CLOUD_API_ACCOUNT_KEY \ + REDIS_CLOUD_API_USER_KEY \ + REDIS_CLOUD_TEST_PRO_SUB_ID \ + REDIS_CLOUD_TEST_PRO_DB_ID \ + REDIS_CLOUD_TEST_ESSENTIALS_SUB_ID \ + REDIS_CLOUD_TEST_ESSENTIALS_DB_ID + do + if [[ -z "${!name:-}" ]]; then + echo "::error title=Missing live-test secret::$name is not configured in the live-contract-tests environment." + missing=1 + fi + done + + for name in \ + REDIS_CLOUD_TEST_PRO_SUB_ID \ + REDIS_CLOUD_TEST_PRO_DB_ID \ + REDIS_CLOUD_TEST_ESSENTIALS_SUB_ID \ + REDIS_CLOUD_TEST_ESSENTIALS_DB_ID + do + if [[ -n "${!name:-}" && "${!name}" == *[!0-9]* ]]; then + echo "::error title=Invalid live-test resource::$name must be a numeric ID." + missing=1 + fi + done + + if [[ -n "${REDIS_CLOUD_ALLOW_DESTRUCTIVE:-}" ]]; then + echo "::error title=Destructive mode is forbidden::This workflow only runs safe and reversible validation." + exit 1 + fi + exit "$missing" + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo + uses: Swatinem/rust-cache@v2 + + - name: Run safe live integration suite + id: live + run: cargo test --test live_integration --all-features -- --ignored --test-threads=1 + + - name: Run compliance baseline gate + id: compliance + env: + COMPLIANCE_SUMMARY_PATH: compliance-summary.txt + run: cargo test --test compliance --all-features api_compliance -- --ignored --exact --test-threads=1 + + - name: Publish payload-free summary + if: always() + shell: bash + env: + LIVE_OUTCOME: ${{ steps.live.outcome }} + COMPLIANCE_OUTCOME: ${{ steps.compliance.outcome }} + run: | + live_outcome="${LIVE_OUTCOME:-not-run}" + compliance_outcome="${COMPLIANCE_OUTCOME:-not-run}" + if [[ -f tests/fixtures/compliance_baseline.json ]]; then + baseline="$(jq -r ' + [to_entries[].value.status] + | group_by(.) + | map("\(.[0])=\(length)") + | join(" ") + ' tests/fixtures/compliance_baseline.json)" + else + baseline="unavailable" + fi + if [[ -f compliance-summary.txt ]]; then + current_compliance="$(tr -d '\n' < compliance-summary.txt)" + else + current_compliance="unavailable" + fi + + { + echo "Redis Cloud live contract validation" + echo "commit=$GITHUB_SHA" + echo "live_integration=$live_outcome" + echo "compliance=$compliance_outcome" + echo "current_compliance=$current_compliance" + echo "committed_baseline=$baseline" + } > live-validation-summary.txt + + { + echo "## Redis Cloud live contract validation" + echo + echo "- Commit: \`$GITHUB_SHA\`" + echo "- Live integration suite: **$live_outcome**" + echo "- Compliance baseline gate: **$compliance_outcome**" + echo "- Current compliance: \`$current_compliance\`" + echo "- Committed baseline: \`$baseline\`" + echo + echo "The artifact contains only this aggregate summary; live API payloads are never uploaded." + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload validation summary + if: ${{ always() && hashFiles('live-validation-summary.txt') != '' }} + uses: actions/upload-artifact@v4 + with: + name: redis-cloud-live-validation + path: live-validation-summary.txt + retention-days: 14 diff --git a/tests/compliance.rs b/tests/compliance.rs index 24136f0..b409735 100644 --- a/tests/compliance.rs +++ b/tests/compliance.rs @@ -254,6 +254,67 @@ async fn record_write( m.insert(key(method, spec_path), status); } +/// Exercise the reversible tag surface without replacing unrelated tags. +/// Failure to snapshot the current collection fails closed: the baseline will +/// see these operations change to `Skip`, and no write is attempted. +async fn run_tag_lifecycle( + m: &mut Matrix, + c: &CloudClient, + live_base: &str, + spec_base: &str, + spec_item: &str, +) { + const TEST_KEY: &str = "rcrs-compliance"; + let preserved = match c.get_raw(live_base).await { + Ok(raw) => raw.get("tags").and_then(Value::as_array).map(|tags| { + tags.iter() + .filter(|tag| tag.get("key").and_then(Value::as_str) != Some(TEST_KEY)) + .cloned() + .collect::>() + }), + Err(error) => { + eprintln!(" [FAIL detail] GET {spec_base}: {error}"); + None + } + }; + let Some(mut replacement) = preserved else { + for (method, path) in [ + ("POST", spec_base), + ("PUT", spec_item), + ("PUT", spec_base), + ("DELETE", spec_item), + ] { + skip(m, method, path, "could not snapshot existing tags safely"); + } + return; + }; + + let live_item = format!("{live_base}/{TEST_KEY}"); + let _ = c.delete_raw(&live_item).await; + + let result = c + .post_raw( + live_base, + serde_json::json!({"key": TEST_KEY, "value": "v1"}), + ) + .await; + record_write::(m, "POST", spec_base, result).await; + + let result = c + .put_raw(&live_item, serde_json::json!({"value": "v2"})) + .await; + record_write::(m, "PUT", spec_item, result).await; + + replacement.push(serde_json::json!({"key": TEST_KEY, "value": "v3"})); + let result = c + .put_raw(live_base, serde_json::json!({"tags": replacement})) + .await; + record_write::(m, "PUT", spec_base, result).await; + + let result = c.delete_raw(&live_item).await; + record_write::(m, "DELETE", spec_item, result).await; +} + /// Auto-classify every write operation not explicitly wired above as a `Skip`, /// with a reason: connectivity/Active-Active ops need resources we don't have; /// everything else is a mutating/destructive op deferred to the systematic @@ -333,6 +394,94 @@ fn bundled_spec_operations_are_base_relative() { assert!(ops.iter().all(|(_, path)| !path.starts_with("/v1/"))); } +#[tokio::test] +async fn tag_lifecycle_preserves_unrelated_tags() { + use wiremock::matchers::{body_json, method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let server = MockServer::start().await; + let live_base = "/subscriptions/1/databases/2/tags"; + let live_item = "/subscriptions/1/databases/2/tags/rcrs-compliance"; + + Mock::given(method("GET")) + .and(path(live_base)) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "tags": [ + {"key": "keep", "value": "original"}, + {"key": "rcrs-compliance", "value": "stale"} + ] + }))) + .expect(1) + .mount(&server) + .await; + Mock::given(method("DELETE")) + .and(path(live_item)) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({}))) + .expect(2) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path(live_base)) + .and(body_json( + serde_json::json!({"key": "rcrs-compliance", "value": "v1"}), + )) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(serde_json::json!({"key": "rcrs-compliance", "value": "v1"})), + ) + .expect(1) + .mount(&server) + .await; + Mock::given(method("PUT")) + .and(path(live_item)) + .and(body_json(serde_json::json!({"value": "v2"}))) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(serde_json::json!({"key": "rcrs-compliance", "value": "v2"})), + ) + .expect(1) + .mount(&server) + .await; + Mock::given(method("PUT")) + .and(path(live_base)) + .and(body_json(serde_json::json!({ + "tags": [ + {"key": "keep", "value": "original"}, + {"key": "rcrs-compliance", "value": "v3"} + ] + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "tags": [ + {"key": "keep", "value": "original"}, + {"key": "rcrs-compliance", "value": "v3"} + ] + }))) + .expect(1) + .mount(&server) + .await; + + let client = CloudClient::builder() + .api_key("test-key") + .api_secret("test-secret") + .base_url(server.uri()) + .build() + .expect("mock client should build"); + let mut matrix = Matrix::new(); + let spec_base = "/subscriptions/{subscriptionId}/databases/{databaseId}/tags"; + let spec_item = "/subscriptions/{subscriptionId}/databases/{databaseId}/tags/{tagKey}"; + + run_tag_lifecycle(&mut matrix, &client, live_base, spec_base, spec_item).await; + + for operation in [ + key("POST", spec_base), + key("PUT", spec_item), + key("PUT", spec_base), + key("DELETE", spec_item), + ] { + assert_eq!(matrix.get(&operation), Some(&Status::Pass)); + } +} + /// Add `Uncovered` for every spec op without a registered check, and detect /// registered ops that don't exist in the spec (typos / non-spec routes). fn reconcile_with_spec(m: &mut Matrix) { @@ -375,7 +524,14 @@ fn print_report(m: &Matrix) { .iter() .map(|k| format!("{}={}", k, counts.get(k).copied().unwrap_or(0))) .collect(); - println!("--- {total} operations: {} ---", summary.join(" ")); + let summary = format!("{total} operations: {}", summary.join(" ")); + println!("--- {summary} ---"); + + // Scheduled CI may request an aggregate-only artifact. This deliberately + // contains no operation paths, response bodies, or captured live values. + if let Ok(path) = std::env::var("COMPLIANCE_SUMMARY_PATH") { + std::fs::write(path, format!("{summary}\n")).expect("write compliance summary"); + } } fn load_baseline() -> Option { @@ -431,6 +587,19 @@ fn env_i32(k: &str) -> Option { std::env::var(k).ok()?.parse().ok() } +fn canonical_test_subscription_name(name: &str) -> String { + let mut name = name.to_string(); + loop { + let Some(stripped) = ["-rcrs-upd-test", "-rcrs-compliance"] + .iter() + .find_map(|suffix| name.strip_suffix(suffix)) + else { + return name; + }; + name = stripped.to_string(); + } +} + // --------------------------------------------------------------------------- // The harness // --------------------------------------------------------------------------- @@ -464,7 +633,7 @@ async fn api_compliance() { AccountSubscriptions, Subscription, SubscriptionMaintenanceWindows, SubscriptionPricings, }; use redis_cloud::types::{ - CloudTag, CloudTags, DatabaseTrafficStateResponse, TaskStateUpdate, TasksStateUpdate, + CloudTags, DatabaseTrafficStateResponse, TaskStateUpdate, TasksStateUpdate, }; use redis_cloud::users::AccountUsers; @@ -990,111 +1159,26 @@ async fn api_compliance() { // -- T2: non-destructive write lifecycles (reversible, self-cleaning) -- - // Database tags (Pro): create -> update one -> overwrite all -> delete. - { - let base = format!("/subscriptions/{ps}/databases/{pd}/tags"); - let _ = c.delete_raw(&format!("{base}/rcrs-compliance")).await; // pre-clean - let r = c - .post_raw( - &base, - serde_json::json!({"key": "rcrs-compliance", "value": "v1"}), - ) - .await; - record_write::( - &mut m, - "POST", - "/subscriptions/{subscriptionId}/databases/{databaseId}/tags", - r, - ) - .await; - let r = c - .put_raw( - &format!("{base}/rcrs-compliance"), - serde_json::json!({"value": "v2"}), - ) - .await; - record_write::( - &mut m, - "PUT", - "/subscriptions/{subscriptionId}/databases/{databaseId}/tags/{tagKey}", - r, - ) - .await; - let r = c - .put_raw( - &base, - serde_json::json!({"tags": [{"key": "rcrs-compliance", "value": "v3"}]}), - ) - .await; - record_write::( - &mut m, - "PUT", - "/subscriptions/{subscriptionId}/databases/{databaseId}/tags", - r, - ) - .await; - let r = c.delete_raw(&format!("{base}/rcrs-compliance")).await; - record_write::( - &mut m, - "DELETE", - "/subscriptions/{subscriptionId}/databases/{databaseId}/tags/{tagKey}", - r, - ) - .await; - } + // Database tags (Pro): create -> update one -> replace all -> delete, + // preserving every pre-existing non-test tag throughout the lifecycle. + run_tag_lifecycle( + &mut m, + &c, + &format!("/subscriptions/{ps}/databases/{pd}/tags"), + "/subscriptions/{subscriptionId}/databases/{databaseId}/tags", + "/subscriptions/{subscriptionId}/databases/{databaseId}/tags/{tagKey}", + ) + .await; - // Database tags (Essentials): same lifecycle. - { - let base = format!("/fixed/subscriptions/{es}/databases/{ed}/tags"); - let _ = c.delete_raw(&format!("{base}/rcrs-compliance")).await; - let r = c - .post_raw( - &base, - serde_json::json!({"key": "rcrs-compliance", "value": "v1"}), - ) - .await; - record_write::( - &mut m, - "POST", - "/fixed/subscriptions/{subscriptionId}/databases/{databaseId}/tags", - r, - ) - .await; - let r = c - .put_raw( - &format!("{base}/rcrs-compliance"), - serde_json::json!({"value": "v2"}), - ) - .await; - record_write::( - &mut m, - "PUT", - "/fixed/subscriptions/{subscriptionId}/databases/{databaseId}/tags/{tagKey}", - r, - ) - .await; - let r = c - .put_raw( - &base, - serde_json::json!({"tags": [{"key": "rcrs-compliance", "value": "v3"}]}), - ) - .await; - record_write::( - &mut m, - "PUT", - "/fixed/subscriptions/{subscriptionId}/databases/{databaseId}/tags", - r, - ) - .await; - let r = c.delete_raw(&format!("{base}/rcrs-compliance")).await; - record_write::( - &mut m, - "DELETE", - "/fixed/subscriptions/{subscriptionId}/databases/{databaseId}/tags/{tagKey}", - r, - ) - .await; - } + // Database tags (Essentials): same preserving lifecycle. + run_tag_lifecycle( + &mut m, + &c, + &format!("/fixed/subscriptions/{es}/databases/{ed}/tags"), + "/fixed/subscriptions/{subscriptionId}/databases/{databaseId}/tags", + "/fixed/subscriptions/{subscriptionId}/databases/{databaseId}/tags/{tagKey}", + ) + .await; // ACL redis rule (account-global): create -> update -> delete. { @@ -1169,12 +1253,13 @@ async fn api_compliance() { } // Subscription update (Pro): rename, then restore. - if let Some(orig) = c + if let Some(observed) = c .get_raw(&format!("/subscriptions/{ps}")) .await .ok() .and_then(|v| v["name"].as_str().map(String::from)) { + let orig = canonical_test_subscription_name(&observed); let r = c .put_raw( &format!("/subscriptions/{ps}"), @@ -1198,12 +1283,13 @@ async fn api_compliance() { } // Subscription update (Essentials): rename, then restore. - if let Some(orig) = c + if let Some(observed) = c .get_raw(&format!("/fixed/subscriptions/{es}")) .await .ok() .and_then(|v| v["name"].as_str().map(String::from)) { + let orig = canonical_test_subscription_name(&observed); let r = c .put_raw( &format!("/fixed/subscriptions/{es}"), diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index 5bad0f0..6c999b0 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -79,6 +79,45 @@ temporarily modify dedicated resources and creates then deletes a test ACL rule. Run it only against the pinned test resources, and outside any sandbox that blocks outbound TLS. +## Scheduled live validation + +The `Redis Cloud Live Contract Validation` workflow runs every Wednesday at +14:37 UTC and can be dispatched manually. It uses the `live-contract-tests` +GitHub environment, whose secrets must point only at a dedicated non-production +account: + +- `REDIS_CLOUD_API_ACCOUNT_KEY` +- `REDIS_CLOUD_API_USER_KEY` +- `REDIS_CLOUD_TEST_PRO_SUB_ID` +- `REDIS_CLOUD_TEST_PRO_DB_ID` +- `REDIS_CLOUD_TEST_ESSENTIALS_SUB_ID` +- `REDIS_CLOUD_TEST_ESSENTIALS_DB_ID` + +The workflow fails before building if any value is absent or a resource ID is +not numeric. It never prints those values, sets `COMPLIANCE_BLESS`, or enables +destructive validation. The live integration and compliance suites run +serially, and overlapping scheduled/manual runs queue instead of cancelling an +in-progress cleanup. No subscription, database, or cloud account is created or +deleted. + +Reversible writes reserve the `rcrs-` marker names. Tag and ACL-rule lifecycles +pre-clean artifacts left by an interrupted run, preserve unrelated tags, and +clean up on completion. Subscription renames restore before assertions and +strip known test suffixes on the next run, so a force-terminated run is +self-healing. If a run is interrupted, rerun it before using the pinned +resources for other work and inspect their names/tags/rules before intervening +manually. + +The compliance baseline is a hard gate: any changed status or dropped-field set +fails the job. The Actions summary and 14-day artifact contain only aggregate +outcomes and committed baseline counts, never captured live payloads. Reproduce +the scheduled commands locally with: + +```bash +cargo test --test live_integration --all-features -- --ignored --test-threads=1 +cargo test --test compliance --all-features api_compliance -- --ignored --exact --test-threads=1 +``` + ## Capturing fixtures for inspection `scripts/generate-cloud-fixtures.sh` captures live responses into the gitignored diff --git a/tests/live_integration.rs b/tests/live_integration.rs index ebb2c4b..4199821 100644 --- a/tests/live_integration.rs +++ b/tests/live_integration.rs @@ -66,6 +66,19 @@ fn env_i32(key: &str) -> Option { std::env::var(key).ok()?.parse().ok() } +fn canonical_test_subscription_name(name: &str) -> String { + let mut name = name.to_string(); + loop { + let Some(stripped) = ["-rcrs-upd-test", "-rcrs-compliance"] + .iter() + .find_map(|suffix| name.strip_suffix(suffix)) + else { + return name; + }; + name = stripped.to_string(); + } +} + fn test_resources() -> Option { Some(TestResources { pro_sub: env_i32("REDIS_CLOUD_TEST_PRO_SUB_ID")?, @@ -551,13 +564,17 @@ live_test_pinned!(live_pro_subscription_update_name, c, res, { } let sub = res.pro_sub; - let original = c + let observed = c .subscriptions() .get_subscription_by_id(sub) .await .expect("get_subscription_by_id should deserialize") .name .expect("test subscription should have a name"); + // A force-terminated prior live/compliance run may have left a reserved + // suffix behind. Always derive the stable name before the new lifecycle so + // this run heals it when restoring. + let original = canonical_test_subscription_name(&observed); let temp = format!("{original}-rcrs-upd-test"); // Rename to a temp value.