Pr/dev first 11 through 2dc73cd

CHANGELOG.md

@@ -11,6 +11,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 - Phase 1 database architecture foundation with workload budgets, telemetry ingest batch diagnostics, reporting watermarks, reconciliation tables, current summary read model, reporting query budgets, one-shot phase runner, manifest, stress scenario, and database architecture docs.
 - Phase 2 analytics scalability with asynchronous recalculation jobs, job status API, idempotent aggregate upserts, bounded summary metadata, analytics cache schema, dashboard queued-job feedback, and contract documentation.
 - Phase 3 edge hot-path performance with worker-local config caching, last-known-good reload behavior, bounded shared-memory telemetry queues, queue health visibility, configurable edge capacity defaults, one-shot phase manifest, and stress scenario registration.
+- Phase 4 real challenge and clearance system for WAF, rate-limit, and Emergency Protection flows, including a self-hosted browser verification page, per-rule challenge difficulty, edge-only verification endpoint, scoped HttpOnly clearance cookies, block-rule precedence, edge secret and difficulty fallback configuration, one-shot phase manifest, e2e coverage, and stress scenario registration.
+- Phase 5 waiting room with persisted per-domain policy, API and dashboard controls, signed edge queue tickets, signed admission cookies, local queue/status endpoints, bounded edge queue state, and operator documentation.
+- Phase 6 cache correctness with explicit eligibility and key-dimension settings, safe debug headers, personalized-request bypass defaults, per-rule edge TTL honoring, cache-status headers, a one-shot phase manifest, and stress scenario registration.
+- Phase 7 origin routing resilience with primary, backup, and shield roles, deterministic weighted selection, drain controls, bounded idempotent retry metadata, circuit-breaker settings, shield visibility, edge-sourced origin health observations, per-edge latency and jitter reporting, one-shot phase manifest, and stress scenario registration.
 - Single canonical roadmap at `docs/ROADMAP.md`; removed duplicate root, lower-case, and legacy roadmap files.
 - Repository-level project presentation, contribution, security, roadmap, changelog, issue template, and pull request template documentation.
 

README.md

@@ -185,6 +185,12 @@ CDNLITE_EDGE_HEALTH_MODE=static ./ci/dns_e2e.sh
 
 Run the destructive DNS stress test only against an explicitly disposable environment.
 
+Useful operator commands include `php artisan cdn:edge:sync-config` to publish or
+fetch the active edge config and `php artisan cdn:config-snapshots:prune
+--keep=2 --batch=5000 --dry-run` to review bounded snapshot cleanup before
+deleting old published artifacts. Snapshot rollback/history endpoints are
+disabled by default because database tables remain the source of truth.
+
 ## Contributing
 
 Contributions are welcome across docs, tests, OpenResty/Lua edge work, Vue dashboard UX, PHP control plane, security hardening, deployment examples, Kubernetes/Helm, Prometheus/Grafana, RBAC, and SSO.

ci/e2e.sh

@@ -601,14 +601,14 @@ record_step PASS "guided-onboarding-flow" "recommendation logic, preview, skip/r
 
 config_snapshot_json="$(docker compose exec -T core php artisan cdn:edge:sync-config)"
 config_snapshot_after="$(jq -r '.version' <<<"$config_snapshot_json")"
-config_snapshot_reused_json="$(docker compose exec -T core php artisan cdn:edge:sync-config)"
-config_snapshot_reused_version="$(jq -r '.version' <<<"$config_snapshot_reused_json")"
-config_snapshot_reused="$(jq -r '.reused // false' <<<"$config_snapshot_reused_json")"
+config_snapshot_cached_json="$(docker compose exec -T core php artisan cdn:edge:sync-config)"
+config_snapshot_cached_version="$(jq -r '.version' <<<"$config_snapshot_cached_json")"
+config_snapshot_cached_reused="$(jq -r '.reused // false' <<<"$config_snapshot_cached_json")"
 retry 20 1 config_publish_audit_exists
 assert_eq "$config_snapshot_after" "$(current_config_state_version)" "config rebuild should activate the published snapshot"
-assert_eq "$config_snapshot_reused_version" "$config_snapshot_after" "unchanged config rebuild should reuse the active snapshot version"
-assert_eq "$config_snapshot_reused" "true" "unchanged config rebuild should report a reused snapshot"
-record_step PASS "config-publish-audit" "config rebuild writes publish audit events and reuses unchanged snapshots"
+assert_eq "$config_snapshot_cached_version" "$config_snapshot_after" "unchanged config fetch should serve the active snapshot version"
+assert_eq "$config_snapshot_cached_reused" "false" "unchanged edge config fetch should not rebuild just to report reused"
+record_step PASS "config-publish-audit" "config publish writes audit events and unchanged fetches use the active snapshot"
 
 api_post_with_powerdns_retry "${CORE_URL}/api/v1/domains/${DOMAIN_ID}/dns/records" \
   '{"type":"A","name":"@","content":"1.1.1.1","ttl":300,"proxied":true,"origin_host":"origin-tls","origin_tls_verify":"ignore","geo_origins":{"DEFAULT":{"host":"origin-tls","tls_verify":"ignore"},"IR":{"host":"origin-http","tls_verify":"verify"}}}'
@@ -952,6 +952,16 @@ assert_contains "$origin_unchecked_unhealthy_body" '"origin_scheme":"http"' "unc
 assert_contains "$origin_unchecked_unhealthy_body" "\"origin_host\":\"${TEST_DOMAIN}\"" "unchecked unhealthy origin should still preserve requested Host"
 record_step PASS "origin-health-disabled-still-routes" "core health_status=unhealthy did not block edge traffic when health_check_enabled=false"
 
+edge_api POST "/api/v1/collector/usage" "{\"idempotency_key\":\"e2e-${RUN_KEY}-origin-health\",\"items\":[{\"ts\":$(date +%s),\"domain_id\":\"${DOMAIN_ID}\",\"edge_node_id\":\"${EDGE_ID}\",\"requests_count\":0,\"bytes_in\":0,\"bytes_out\":0,\"status\":200,\"method\":\"HEALTH\",\"path\":\"/health\",\"host\":\"${TEST_DOMAIN}\",\"origin_id\":\"${PRIMARY_ORIGIN_ID}\",\"origin_host\":\"${ORIGIN_HTTP_IP}\",\"upstream_status\":\"200\",\"upstream_response_time\":\"3.250\",\"router_error\":\"\",\"origin_health_probe\":true},{\"ts\":$(date +%s),\"domain_id\":\"${DOMAIN_ID}\",\"edge_node_id\":\"${EDGE_ID}\",\"requests_count\":0,\"bytes_in\":0,\"bytes_out\":0,\"status\":200,\"method\":\"HEALTH\",\"path\":\"/health\",\"host\":\"${TEST_DOMAIN}\",\"origin_id\":\"${PRIMARY_ORIGIN_ID}\",\"origin_host\":\"${ORIGIN_HTTP_IP}\",\"upstream_status\":\"200\",\"upstream_response_time\":\"1.000\",\"router_error\":\"\",\"origin_health_probe\":true}]}"
+assert_http_status "$HTTP_CODE" "200" "edge origin health observation ingest failed"
+api_get "${CORE_URL}/api/v1/domains/${DOMAIN_ID}/origins/health"
+assert_http_status "$HTTP_CODE" "200" "origin health report failed"
+assert_contains "$HTTP_BODY" '"source":"edge_observations"' "origin health report should be edge-sourced"
+assert_contains "$HTTP_BODY" '"core_active_checks":false' "core active checks should not be authoritative"
+assert_contains "$HTTP_BODY" '"status":"slow"' "slow origin observation should be visible"
+assert_contains "$HTTP_BODY" '"jitter_ms":2250' "origin jitter should be derived from edge observations"
+record_step PASS "origin-health-edge-observations" "edge-sourced origin health report includes slow and jitter details"
+
 ORIGIN_TLS_IP="$(docker compose exec -T edge sh -lc "getent hosts origin-tls | awk '{print \$1; exit}'" | tr -d '\r')"
 if [[ -z "$ORIGIN_TLS_IP" ]]; then
   fail "unable to resolve origin-tls container IP from edge"
@@ -1224,7 +1234,7 @@ record_step PASS "edge-waf-block-runtime" "403 and edge header observed for /adm
 bot_challenge_body="$(mktemp)"
 bot_challenge_status="$(curl -sS -o "$bot_challenge_body" -w '%{http_code}' -H "Host: ${TEST_DOMAIN}" -A 'Googlebot' "${EDGE_URL}/bot-check?via=edge-bot-challenge")"
 assert_eq "$bot_challenge_status" "403" "unverified search-bot claim should be challenged"
-assert_contains "$(cat "$bot_challenge_body")" '"error":"bot_challenge_required"' "bot challenge response should be explicit"
+assert_contains "$(cat "$bot_challenge_body")" "Security check" "bot challenge should return the browser verification page"
 rm -f "$bot_challenge_body"
 retry 10 1 agent_exec '/agent/push_security_events.sh'
 bot_event_visible=0
@@ -1242,7 +1252,7 @@ record_step PASS "edge-bot-protection-runtime" "unverified crawler challenge and
 verified_bot_source_id="verified-bot-${RUN_KEY}"
 now="$(date +%s)"
 db_query "INSERT INTO verified_bot_sources (id,domain_id,bot_class,provider,user_agent_pattern,cidr,enabled,created_at,updated_at) VALUES ('${verified_bot_source_id}','${DOMAIN_ID}','verified_search_bot','Google','Googlebot','0.0.0.0/0',true,${now},${now});" >/dev/null
-db_query "UPDATE config_state SET active_snapshot_version=NULL WHERE id=1;" >/dev/null
+db_query "UPDATE config_state SET dirty=true, dirty_at=${now} WHERE id=1;" >/dev/null
 edge_pull_config
 verified_bot_status="$(curl -sS -o /tmp/e2e-verified-bot-body.txt -w '%{http_code}' -H "Host: ${TEST_DOMAIN}" -A 'Googlebot' "${EDGE_URL}/bot-check?via=edge-verified-bot")"
 assert_eq "$verified_bot_status" "200" "verified search-bot source should allow matching crawler traffic"
@@ -1262,30 +1272,74 @@ edge_id_header_rl="$(edge_header_for_host "${TEST_DOMAIN}" "/login?via=edge-rate
 assert_eq "$edge_id_header_rl" "$EDGE_ID" "rate-limit response should expose edge id header"
 record_step PASS "edge-rate-limit-runtime" "429 observed for /login after burst"
 
-api_post "${CORE_URL}/api/v1/domains/${DOMAIN_ID}/rate-limits" '{"enabled":true,"requests_per_minute":5,"path_prefix":"/challenge","key_type":"ip_path","priority":21,"action":"challenge"}'
+api_post "${CORE_URL}/api/v1/domains/${DOMAIN_ID}/rate-limits" '{"enabled":true,"requests_per_minute":5,"path_prefix":"/challenge","key_type":"ip_path","priority":21,"action":"challenge","challenge_difficulty":3}'
 assert_http_status "$HTTP_CODE" "201" "challenge rate-limit create failed"
 CHALLENGE_RATE_LIMIT_RULE_ID="$(json_get "$HTTP_BODY" '.data.id')"
+assert_contains "$HTTP_BODY" '"challenge_difficulty":3' "challenge rate-limit response should include per-rule difficulty"
 # The edge serves its local snapshot, so publish and apply this new rule before exercising it.
 edge_pull_config
 challenge_codes=()
 challenge_response_verified=0
+challenge_body="/tmp/e2e-rate-limit-challenge-body.txt"
 for i in $(seq 1 8); do
-  code="$(curl -sS -o /tmp/e2e-rate-limit-challenge-${i}.txt -w '%{http_code}' -H "Host: ${TEST_DOMAIN}" "${EDGE_URL}/challenge?via=edge-rate-limit-challenge")"
+  code="$(curl -sS -o "$challenge_body" -w '%{http_code}' -H "Host: ${TEST_DOMAIN}" "${EDGE_URL}/challenge?via=edge-rate-limit-challenge")"
   challenge_codes+=("$code")
-  if [[ "$code" == "429" ]] && grep -Fq '"error":"challenge_required"' "/tmp/e2e-rate-limit-challenge-${i}.txt"; then
+  if [[ "$code" == "429" ]] && grep -Fq "Security check" "$challenge_body" && grep -Fq "__cdnlite_challenge_verify" "$challenge_body"; then
     challenge_response_verified=1
+    break
   fi
 done
 if [[ " ${challenge_codes[*]} " != *" 429 "* ]]; then
   fail "challenge rate limit expected 429 challenge responses"
 fi
-assert_eq "$challenge_response_verified" "1" "challenge rate-limit response should identify the required challenge"
+assert_eq "$challenge_response_verified" "1" "challenge rate-limit response should return browser verification page"
+challenge_token="$(php -r '$body=file_get_contents($argv[1]); if(preg_match("/const token = \"([^\"]+)\";/", $body, $m)){ echo $m[1]; }' "$challenge_body")"
+[[ -n "$challenge_token" ]] || fail "challenge page did not include a signed token"
+challenge_difficulty="$(php -r '$body=file_get_contents($argv[1]); if(preg_match("/const difficulty = ([0-9]+);/", $body, $m)){ echo $m[1]; }' "$challenge_body")"
+challenge_difficulty="${challenge_difficulty:-3}"
+if [[ "$challenge_difficulty" -le 1 ]]; then
+  challenge_pow="browser-check"
+else
+  challenge_pow="$(php -r '$t=$argv[1]; $d=(int)$argv[2]; $prefix=str_repeat("0",$d); for($i=0;$i<5000000;$i++){ if(substr(hash("sha256",$t.":".$i),0,$d)===$prefix){ echo $i; exit(0);} } exit(1);' "$challenge_token" "$challenge_difficulty")"
+fi
+[[ -n "$challenge_pow" ]] || fail "could not solve local challenge proof"
+cookie_jar="/tmp/e2e-cdnlite-clearance-cookie.txt"
+verify_code="$(curl -sS -o /tmp/e2e-challenge-verify.txt -c "$cookie_jar" -w '%{http_code}' -H "Host: ${TEST_DOMAIN}" -X POST "${EDGE_URL}/__cdnlite_challenge_verify" --data-urlencode "token=${challenge_token}" --data-urlencode "pow=${challenge_pow}")"
+assert_eq "$verify_code" "303" "valid challenge proof should set clearance and redirect"
+assert_contains "$(cat "$cookie_jar")" "__cdnlite_clearance" "challenge verification should set clearance cookie"
+cleared_code="$(curl -sS -b "$cookie_jar" -o /tmp/e2e-rate-limit-cleared.txt -w '%{http_code}' -H "Host: ${TEST_DOMAIN}" "${EDGE_URL}/challenge?via=edge-rate-limit-cleared")"
+assert_eq "$cleared_code" "200" "valid clearance should allow matching challenge traffic to origin"
+assert_contains "$(cat /tmp/e2e-rate-limit-cleared.txt)" '"origin_scheme":"https"' "cleared challenge traffic should reach origin"
 # Security events are delivered asynchronously by the edge agent. The event records
 # the configured decision (`challenge`), while the HTTP response says `challenge_required`.
 retry 20 1 challenge_security_event_visible
 api_delete "${CORE_URL}/api/v1/domains/${DOMAIN_ID}/rate-limits/${CHALLENGE_RATE_LIMIT_RULE_ID}"
 assert_http_status "$HTTP_CODE" "200" "challenge rate-limit cleanup failed"
-record_step PASS "edge-rate-limit-challenge" "challenge action returns 429 and is visible in security events"
+record_step PASS "edge-rate-limit-challenge" "challenge page, proof verification, clearance cookie, origin routing, and events verified"
+
+api_post "${CORE_URL}/api/v1/domains/${DOMAIN_ID}/waf-rules" '{"enabled":true,"name":"browser-check-only","priority":22,"type":"path_prefix","pattern":"/browser-check","action":"challenge","challenge_difficulty":1,"description":"verify lightweight challenge difficulty"}'
+assert_http_status "$HTTP_CODE" "201" "difficulty-1 waf challenge create failed"
+DIFFICULTY_ONE_WAF_RULE_ID="$(json_get "$HTTP_BODY" '.data.id')"
+assert_contains "$HTTP_BODY" '"challenge_difficulty":1' "difficulty-1 waf response should include challenge_difficulty"
+edge_pull_config
+difficulty_one_body="/tmp/e2e-waf-difficulty-one.txt"
+difficulty_one_code="$(curl -sS -o "$difficulty_one_body" -w '%{http_code}' -H "Host: ${TEST_DOMAIN}" "${EDGE_URL}/browser-check?via=edge-waf-difficulty-one")"
+assert_eq "$difficulty_one_code" "403" "difficulty-1 waf rule should serve challenge page"
+assert_contains "$(cat "$difficulty_one_body")" "Security check" "difficulty-1 waf should return challenge page"
+difficulty_one_value="$(php -r '$body=file_get_contents($argv[1]); if(preg_match("/const difficulty = ([0-9]+);/", $body, $m)){ echo $m[1]; }' "$difficulty_one_body")"
+assert_eq "$difficulty_one_value" "1" "difficulty-1 waf challenge page should render difficulty 1"
+difficulty_one_token="$(php -r '$body=file_get_contents($argv[1]); if(preg_match("/const token = \"([^\"]+)\";/", $body, $m)){ echo $m[1]; }' "$difficulty_one_body")"
+[[ -n "$difficulty_one_token" ]] || fail "difficulty-1 challenge page did not include token"
+difficulty_one_cookie="/tmp/e2e-difficulty-one-cookie.txt"
+difficulty_one_verify="$(curl -sS -o /tmp/e2e-difficulty-one-verify.txt -c "$difficulty_one_cookie" -w '%{http_code}' -H "Host: ${TEST_DOMAIN}" -X POST "${EDGE_URL}/__cdnlite_challenge_verify" --data-urlencode "token=${difficulty_one_token}" --data-urlencode "pow=browser-check")"
+assert_eq "$difficulty_one_verify" "303" "difficulty-1 browser check should set clearance"
+assert_contains "$(cat "$difficulty_one_cookie")" "__cdnlite_clearance" "difficulty-1 browser check should set clearance cookie"
+difficulty_one_cleared="$(curl -sS -b "$difficulty_one_cookie" -o /tmp/e2e-difficulty-one-cleared.txt -w '%{http_code}' -H "Host: ${TEST_DOMAIN}" "${EDGE_URL}/browser-check?via=edge-waf-difficulty-one-cleared")"
+assert_eq "$difficulty_one_cleared" "200" "difficulty-1 clearance should allow origin routing"
+assert_contains "$(cat /tmp/e2e-difficulty-one-cleared.txt)" '"origin_scheme":"https"' "difficulty-1 cleared traffic should reach origin"
+api_delete "${CORE_URL}/api/v1/domains/${DOMAIN_ID}/waf-rules/${DIFFICULTY_ONE_WAF_RULE_ID}"
+assert_http_status "$HTTP_CODE" "200" "difficulty-1 waf cleanup failed"
+record_step PASS "edge-waf-challenge-difficulty" "path-level WAF challenge difficulty 1 used browser check and reached origin after clearance"
 
 if docker compose exec -T edge-agent sh -lc "grep -q 'rate_limited' \"\${METRIC_PATH:-/var/lib/cdnlite/metrics.ndjson}\"" \
   || docker compose exec -T edge sh -lc "grep -q 'rate_limited' /var/lib/cdnlite/metrics.ndjson"; then
@@ -1508,7 +1562,8 @@ record_step PASS "edge-config-fetch" "version=${cfg_version}"
 
 edge_api GET "/api/v1/edge/config?if_version=${cfg_version}" ""
 assert_http_status "$HTTP_CODE" "200" "edge config if_version failed"
-record_step PASS "edge-config-if-version" "if_version request ok"
+assert_contains "$HTTP_BODY" '"not_modified":true' "edge config if_version should return not_modified"
+record_step PASS "edge-config-if-version" "if_version request returned not_modified"
 
 miss_code="$(curl -s -o /tmp/e2e-miss-auth.txt -w '%{http_code}' -X POST "${CORE_URL}/api/v1/edge/heartbeat" -H 'Content-Type: application/json' -d "{\"edge_id\":\"${EDGE_ID}\"}")"
 assert_eq "$miss_code" "401" "missing auth should 401"

ci/phase.sh

@@ -48,6 +48,22 @@ case "$PHASE" in
     PHASE_CONTRACT="core/tests/test_phase3_edge_hot_path_contract.py"
     PHASE_STRESS_SCENARIO="phase3-edge-hot-path"
     ;;
+  04)
+    PHASE_CONTRACT="core/tests/test_phase4_challenge_clearance_contract.py"
+    PHASE_STRESS_SCENARIO="phase4-challenge-clearance"
+    ;;
+  05)
+    PHASE_CONTRACT="core/tests/test_phase5_waiting_room_contract.py"
+    PHASE_STRESS_SCENARIO="phase5-waiting-room"
+    ;;
+  06)
+    PHASE_CONTRACT="core/tests/test_phase6_cache_correctness_contract.py"
+    PHASE_STRESS_SCENARIO="phase6-cache-correctness"
+    ;;
+  07)
+    PHASE_CONTRACT="core/tests/test_phase7_origin_resilience_contract.py"
+    PHASE_STRESS_SCENARIO="phase7-origin-resilience"
+    ;;
   *)
     fail "phase ${PHASE} is not registered yet"
     ;;