From abf645b778cec637bc52f15dec421f55acab1a33 Mon Sep 17 00:00:00 2001 From: Gracecom <64633490+Gracecom1@users.noreply.github.com> Date: Thu, 30 Jul 2026 15:31:00 +0900 Subject: [PATCH 1/3] ops: verify live Video Factory production readiness --- .../verify-video-factory-production-ready.yml | 206 ++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 .github/workflows/verify-video-factory-production-ready.yml diff --git a/.github/workflows/verify-video-factory-production-ready.yml b/.github/workflows/verify-video-factory-production-ready.yml new file mode 100644 index 000000000..e8ef8f63f --- /dev/null +++ b/.github/workflows/verify-video-factory-production-ready.yml @@ -0,0 +1,206 @@ +name: Verify Video Factory Production Ready + +on: + pull_request: + paths: + - .github/workflows/verify-video-factory-production-ready.yml + workflow_dispatch: + +permissions: + contents: read + +jobs: + verify: + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + COOLIFY_API_TOKEN: ${{ secrets.COOLIFY_TOKEN }} + COOLIFY_API_URL: https://coolify.paradigmjp.com + PARADIGM_APP_UUID: n8i2sjiqvr2d8hrzppop2m2i + SITE: https://www.paradigmjp.com + steps: + - uses: actions/checkout@v4 + + - name: Create a short-lived administrator session + run: | + set -euo pipefail + envs="$(curl --fail-with-body --silent --show-error \ + -H "Authorization: Bearer ${COOLIFY_API_TOKEN}" \ + -H 'Accept: application/json' \ + "${COOLIFY_API_URL%/}/api/v1/applications/${PARADIGM_APP_UUID}/envs")" + secret="$(printf '%s' "${envs}" | jq -r ' + def active($name): + [ .[]? + | select((.is_preview // false) == false) + | select(.key == $name) + | (.real_value // .value // empty) + ] | map(select(type == "string" and length >= 16)) | .[0] // empty; + active("ADMIN_SESSION_SECRET") + // active("ADMIN_PASSWORD") + // active("PAYLOAD_SECRET") + ')" + [ -n "${secret}" ] || { echo 'No administrator session secret is configured' >&2; exit 1; } + ADMIN_SESSION_SECRET="${secret}" node <<'NODE' > /tmp/admin-cookie.txt + const crypto = require('node:crypto') + const secret = process.env.ADMIN_SESSION_SECRET + const expiresAt = Math.floor(Date.now() / 1000) + 1800 + const nonce = crypto.randomBytes(18).toString('base64url') + const payload = `${expiresAt}.${nonce}` + const signature = crypto.createHmac('sha256', secret).update(payload).digest('base64url') + process.stdout.write(`paradigm_admin_token=${payload}.${signature}`) + NODE + chmod 600 /tmp/admin-cookie.txt + + - name: Fetch production state + run: | + set -euo pipefail + cookie="$(cat /tmp/admin-cookie.txt)" + fetch() { + local path="$1" output="$2" + curl --fail-with-body --silent --show-error --max-time 90 \ + -H 'Accept: application/json' \ + -H "Cookie: ${cookie}" \ + "${SITE}${path}" > "${output}" + } + fetch '/api/video-factory/v1/console/bootstrap' /tmp/factory-bootstrap.json + fetch '/api/video-factory/v1/runtime' /tmp/factory-runtime.json + fetch '/api/video-factory/v1/registry' /tmp/factory-registry.json + fetch '/api/video-factory/v1/vast/instances' /tmp/factory-instances.json + curl --fail-with-body --silent --show-error --max-time 90 \ + -H 'Accept: application/json' \ + "${SITE}/api/video-factory/runtime-handoff?action=status" \ + > /tmp/factory-activation.json + + - name: Assert real production readiness + run: | + set -euo pipefail + python <<'PY' + import json + from pathlib import Path + + def load(name): + return json.loads(Path(f'/tmp/{name}.json').read_text()) + + bootstrap = load('factory-bootstrap') + runtime = load('factory-runtime') + registry = load('factory-registry') + instances = load('factory-instances') + activation = load('factory-activation') + + if not bootstrap.get('ok'): + raise SystemExit('Video Factory console bootstrap is not ready') + + runtime_data = runtime.get('runtime') or {} + effective = runtime.get('effective_comfyui') or {} + if not runtime.get('ok'): + raise SystemExit('Runtime endpoint is not ready') + if not (runtime_data.get('comfyui_base_url') or effective.get('base_url')): + raise SystemExit('ComfyUI base URL is not configured') + if not ( + runtime_data.get('comfyui_api_key_configured') + or effective.get('api_key_configured') + ): + raise SystemExit('ComfyUI API key is not configured') + vast = runtime.get('vast') or bootstrap.get('vast') or {} + if not vast.get('configured'): + raise SystemExit('Vast.ai credential is not configured') + + contracts = registry.get('contracts') or [] + workflow = next( + (item for item in contracts if item.get('id') == 'abstract-broll-t2v'), + None, + ) + if not workflow: + raise SystemExit('abstract-broll-t2v is missing') + if not workflow.get('enabled'): + raise SystemExit('abstract-broll-t2v is not enabled') + if not workflow.get('workflow_valid') or not workflow.get('file_exists'): + raise SystemExit('abstract-broll-t2v is not valid on disk') + + model_state = registry.get('models') or {} + raw_models = model_state.get('items') if isinstance(model_state, dict) else None + if not isinstance(raw_models, list): + raw_models = model_state if isinstance(model_state, list) else [] + approved = [ + item for item in raw_models + if isinstance(item, dict) + and str(item.get('commercial_use', '')).lower() == 'approved' + ] + if len(approved) < 3: + raise SystemExit(f'Expected at least 3 approved production models, found {len(approved)}') + + rows = instances.get('instances') or instances.get('results') or [] + if isinstance(rows, dict): + rows = [rows] + workers = [ + row for row in rows + if isinstance(row, dict) + and str(row.get('label') or row.get('name') or '').startswith('paradigm-comfyui-wan22-') + ] + running = [ + row for row in workers + if str(row.get('actual_status') or row.get('status') or '').lower() in {'running', 'loading'} + ] + if len(running) != 1: + raise SystemExit(f'Expected exactly one active Paradigm GPU worker, found {len(running)}') + + state = activation.get('state') or activation.get('status') or {} + if not activation.get('ok') or not isinstance(state, dict): + raise SystemExit('Activation state is unavailable') + if not state.get('completed_at') or not state.get('instance_id'): + raise SystemExit('Production activation is not complete') + if int(state['instance_id']) != int(running[0].get('id') or running[0].get('instance_id')): + raise SystemExit('Runtime activation and Vast.ai instance do not match') + + doctor = bootstrap.get('doctor') or {} + doctor_text = json.dumps(doctor).lower() + if 'production_ready' in doctor_text and 'not ready' in doctor_text: + raise SystemExit('Doctor still reports production_ready as not ready') + + evidence = { + 'verified_at': __import__('datetime').datetime.now(__import__('datetime').UTC).isoformat(), + 'factory_api': 'ready', + 'vast_credential': 'configured', + 'comfyui': 'connected', + 'active_gpu_workers': 1, + 'gpu_name': running[0].get('gpu_name'), + 'instance_id': int(state['instance_id']), + 'approved_models': len(approved), + 'workflow': 'abstract-broll-t2v', + 'workflow_enabled': True, + 'activation_completed_at': state['completed_at'], + } + Path('/tmp/video-factory-production-ready.json').write_text( + json.dumps(evidence, indent=2) + '\n' + ) + print(json.dumps(evidence)) + PY + + - name: Verify public console and runtime health + run: | + set -euo pipefail + cookie="$(cat /tmp/admin-cookie.txt)" + console_code="$(curl --silent --show-error --output /tmp/video-factory-console.html \ + --write-out '%{http_code}' --max-time 60 \ + -H 'Accept: text/html' -H "Cookie: ${cookie}" \ + "${SITE}/video-factory-console")" + [ "${console_code}" = '200' ] + grep -q 'Video Factory' /tmp/video-factory-console.html + ready_code="$(curl --silent --show-error --output /tmp/video-factory-ready.json \ + --write-out '%{http_code}' --max-time 60 \ + "${SITE}/api/video-factory/ready")" + [ "${ready_code}" = '200' ] + jq -e '.ready == true and .service == "video-factory"' /tmp/video-factory-ready.json >/dev/null + + - name: Remove administrator session material + if: always() + run: rm -f /tmp/admin-cookie.txt + + - name: Upload sanitized verification evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: video-factory-production-ready + path: /tmp/video-factory-production-ready.json + if-no-files-found: warn + retention-days: 7 From b32ba45d8c701ef43796aabe4e0b2a2407f98552 Mon Sep 17 00:00:00 2001 From: Gracecom <64633490+Gracecom1@users.noreply.github.com> Date: Fri, 31 Jul 2026 06:33:06 +0900 Subject: [PATCH 2/3] fix: verify current production routes and work redirects --- .../verify-video-factory-production-ready.yml | 179 ++++++++++++++---- 1 file changed, 145 insertions(+), 34 deletions(-) diff --git a/.github/workflows/verify-video-factory-production-ready.yml b/.github/workflows/verify-video-factory-production-ready.yml index e8ef8f63f..c3b42eb7a 100644 --- a/.github/workflows/verify-video-factory-production-ready.yml +++ b/.github/workflows/verify-video-factory-production-ready.yml @@ -40,6 +40,7 @@ jobs: // active("PAYLOAD_SECRET") ')" [ -n "${secret}" ] || { echo 'No administrator session secret is configured' >&2; exit 1; } + echo "::add-mask::${secret}" ADMIN_SESSION_SECRET="${secret}" node <<'NODE' > /tmp/admin-cookie.txt const crypto = require('node:crypto') const secret = process.env.ADMIN_SESSION_SECRET @@ -51,31 +52,35 @@ jobs: NODE chmod 600 /tmp/admin-cookie.txt - - name: Fetch production state + - name: Fetch authenticated production state run: | set -euo pipefail cookie="$(cat /tmp/admin-cookie.txt)" fetch() { local path="$1" output="$2" - curl --fail-with-body --silent --show-error --max-time 90 \ + local code + code="$(curl --silent --show-error --max-time 90 \ + --output "${output}" --write-out '%{http_code}' \ -H 'Accept: application/json' \ -H "Cookie: ${cookie}" \ - "${SITE}${path}" > "${output}" + "${SITE}${path}")" + if [ "${code}" != '200' ]; then + echo "Production state fetch failed for ${path} with HTTP ${code}" >&2 + cat "${output}" >&2 || true + exit 1 + fi } fetch '/api/video-factory/v1/console/bootstrap' /tmp/factory-bootstrap.json fetch '/api/video-factory/v1/runtime' /tmp/factory-runtime.json fetch '/api/video-factory/v1/registry' /tmp/factory-registry.json fetch '/api/video-factory/v1/vast/instances' /tmp/factory-instances.json - curl --fail-with-body --silent --show-error --max-time 90 \ - -H 'Accept: application/json' \ - "${SITE}/api/video-factory/runtime-handoff?action=status" \ - > /tmp/factory-activation.json - name: Assert real production readiness run: | set -euo pipefail python <<'PY' import json + from datetime import UTC, datetime from pathlib import Path def load(name): @@ -85,11 +90,26 @@ jobs: runtime = load('factory-runtime') registry = load('factory-registry') instances = load('factory-instances') - activation = load('factory-activation') if not bootstrap.get('ok'): raise SystemExit('Video Factory console bootstrap is not ready') + doctor = bootstrap.get('doctor') or {} + if doctor.get('production_ready') is not True: + reasons = doctor.get('blocking_reasons') or ['unknown production readiness failure'] + raise SystemExit(f"Video Factory doctor is not production-ready: {reasons}") + + comfy = doctor.get('comfyui') or {} + if comfy.get('reachable') is not True: + raise SystemExit('ComfyUI is not reachable from the production service') + if comfy.get('authenticated') is not True: + raise SystemExit('ComfyUI authentication is not configured') + if comfy.get('vram_ready') is not True: + raise SystemExit('ComfyUI GPU VRAM is below the production requirement') + gpu_devices = comfy.get('gpu_devices') or [] + if not gpu_devices: + raise SystemExit('No production GPU device is reported by ComfyUI') + runtime_data = runtime.get('runtime') or {} effective = runtime.get('effective_comfyui') or {} if not runtime.get('ok'): @@ -139,36 +159,36 @@ jobs: ] running = [ row for row in workers - if str(row.get('actual_status') or row.get('status') or '').lower() in {'running', 'loading'} + if str(row.get('actual_status') or row.get('status') or '').lower() == 'running' + ] + active = [ + row for row in workers + if str(row.get('actual_status') or row.get('status') or '').lower() + in {'running', 'loading', 'created', 'starting'} ] if len(running) != 1: - raise SystemExit(f'Expected exactly one active Paradigm GPU worker, found {len(running)}') - - state = activation.get('state') or activation.get('status') or {} - if not activation.get('ok') or not isinstance(state, dict): - raise SystemExit('Activation state is unavailable') - if not state.get('completed_at') or not state.get('instance_id'): - raise SystemExit('Production activation is not complete') - if int(state['instance_id']) != int(running[0].get('id') or running[0].get('instance_id')): - raise SystemExit('Runtime activation and Vast.ai instance do not match') + raise SystemExit(f'Expected exactly one running Paradigm GPU worker, found {len(running)}') + if len(active) != 1: + raise SystemExit(f'Expected no duplicate active Paradigm GPU workers, found {len(active)}') - doctor = bootstrap.get('doctor') or {} - doctor_text = json.dumps(doctor).lower() - if 'production_ready' in doctor_text and 'not ready' in doctor_text: - raise SystemExit('Doctor still reports production_ready as not ready') + instance = running[0] + instance_id = instance.get('id') or instance.get('instance_id') + if not instance_id: + raise SystemExit('Running Vast.ai instance has no instance ID') evidence = { - 'verified_at': __import__('datetime').datetime.now(__import__('datetime').UTC).isoformat(), + 'verified_at': datetime.now(UTC).isoformat(), 'factory_api': 'ready', + 'doctor_production_ready': True, 'vast_credential': 'configured', - 'comfyui': 'connected', + 'comfyui': 'authenticated-and-reachable', 'active_gpu_workers': 1, - 'gpu_name': running[0].get('gpu_name'), - 'instance_id': int(state['instance_id']), + 'gpu_name': instance.get('gpu_name') or gpu_devices[0].get('name'), + 'instance_id': int(instance_id), 'approved_models': len(approved), 'workflow': 'abstract-broll-t2v', 'workflow_enabled': True, - 'activation_completed_at': state['completed_at'], + 'runtime_updated_at': runtime_data.get('updated_at'), } Path('/tmp/video-factory-production-ready.json').write_text( json.dumps(evidence, indent=2) + '\n' @@ -176,25 +196,116 @@ jobs: print(json.dumps(evidence)) PY - - name: Verify public console and runtime health + - name: Verify public-host redirects and authenticated consoles run: | set -euo pipefail cookie="$(cat /tmp/admin-cookie.txt)" - console_code="$(curl --silent --show-error --output /tmp/video-factory-console.html \ - --write-out '%{http_code}' --max-time 60 \ - -H 'Accept: text/html' -H "Cookie: ${cookie}" \ + + unauth_code="$(curl --silent --show-error --max-time 60 \ + --dump-header /tmp/work-unauthenticated.headers \ + --output /dev/null \ + --write-out '%{http_code}' \ + "${SITE}/work")" + case "${unauth_code}" in + 301|302|303|307|308) ;; + *) echo "Unexpected unauthenticated /work HTTP ${unauth_code}" >&2; exit 1 ;; + esac + unauth_location="$(awk 'BEGIN{IGNORECASE=1} /^location:/ {sub(/^[^:]*:[[:space:]]*/, ""); sub(/\r$/, ""); print; exit}' /tmp/work-unauthenticated.headers)" + printf '%s' "${unauth_location}" | grep -q '/admin/login?redirect=%2Fwork' + ! printf '%s' "${unauth_location}" | grep -Eqi '(^|[/:])0\.0\.0\.0([/:]|$)' + + session_code="$(curl --silent --show-error --max-time 60 \ + --dump-header /tmp/work-session.headers \ + --cookie "${cookie}" \ + --cookie-jar /tmp/work-session.cookies \ + --output /dev/null \ + --write-out '%{http_code}' \ + "${SITE}/work/session?redirect=%2Fwork")" + [ "${session_code}" = '307' ] + session_location="$(awk 'BEGIN{IGNORECASE=1} /^location:/ {sub(/^[^:]*:[[:space:]]*/, ""); sub(/\r$/, ""); print; exit}' /tmp/work-session.headers)" + [ "${session_location}" = '/work' ] + ! grep -Eqi '(^|[/:])0\.0\.0\.0([/:]|$)' /tmp/work-session.headers + grep -Eqi '^set-cookie:[[:space:]]*paradigm_work_api_token=' /tmp/work-session.headers + grep -Eqi '^set-cookie:.*[;[:space:]]Secure([;[:space:]]|$)' /tmp/work-session.headers + grep -Eqi '^set-cookie:.*[;[:space:]]HttpOnly([;[:space:]]|$)' /tmp/work-session.headers + + work_cookie="$(awk '$6 == "paradigm_work_api_token" {print $6 "=" $7; exit}' /tmp/work-session.cookies)" + [ -n "${work_cookie}" ] + combined_cookie="${cookie}; ${work_cookie}" + + work_code="$(curl --silent --show-error --max-time 90 \ + --dump-header /tmp/work.headers \ + --cookie "${combined_cookie}" \ + --output /tmp/work.html \ + --write-out '%{http_code}' \ + "${SITE}/work")" + [ "${work_code}" = '200' ] + ! grep -Eqi '(^|[/:])0\.0\.0\.0([/:]|$)' /tmp/work.headers + grep -q 'Paradigm Revenue Operations' /tmp/work.html + grep -q 'Evidence-first Outreach Workbench' /tmp/work.html + + work_api_code="$(curl --silent --show-error --max-time 90 \ + --cookie "${combined_cookie}" \ + -H 'Accept: application/json' \ + --output /tmp/work-api.json \ + --write-out '%{http_code}' \ + "${SITE}/api/work?page=1&pageSize=1")" + [ "${work_api_code}" = '200' ] + jq -e '.ok == true and (.items | type == "array")' /tmp/work-api.json >/dev/null + + blocked_code="$(curl --silent --show-error --max-time 60 \ + --dump-header /tmp/work-open-redirect.headers \ + --cookie "${cookie}" \ + --output /dev/null \ + --write-out '%{http_code}' \ + "${SITE}/work/session?redirect=https%3A%2F%2Fevil.example")" + [ "${blocked_code}" = '307' ] + blocked_location="$(awk 'BEGIN{IGNORECASE=1} /^location:/ {sub(/^[^:]*:[[:space:]]*/, ""); sub(/\r$/, ""); print; exit}' /tmp/work-open-redirect.headers)" + [ "${blocked_location}" = '/work' ] + + admin_code="$(curl --silent --show-error --max-time 60 \ + --dump-header /tmp/admin-video-factory.headers \ + --cookie "${cookie}" \ + --output /dev/null \ + --write-out '%{http_code}' \ + "${SITE}/admin/video-factory")" + [ "${admin_code}" = '307' ] + admin_location="$(awk 'BEGIN{IGNORECASE=1} /^location:/ {sub(/^[^:]*:[[:space:]]*/, ""); sub(/\r$/, ""); print; exit}' /tmp/admin-video-factory.headers)" + [ "${admin_location}" = '/video-factory-console' ] + ! grep -Eqi '(^|[/:])0\.0\.0\.0([/:]|$)' /tmp/admin-video-factory.headers + + console_code="$(curl --silent --show-error --max-time 90 \ + --cookie "${cookie}" \ + -H 'Accept: text/html' \ + --output /tmp/video-factory-console.html \ + --write-out '%{http_code}' \ "${SITE}/video-factory-console")" [ "${console_code}" = '200' ] grep -q 'Video Factory' /tmp/video-factory-console.html - ready_code="$(curl --silent --show-error --output /tmp/video-factory-ready.json \ - --write-out '%{http_code}' --max-time 60 \ + + ready_code="$(curl --silent --show-error --max-time 60 \ + --output /tmp/video-factory-ready.json \ + --write-out '%{http_code}' \ "${SITE}/api/video-factory/ready")" [ "${ready_code}" = '200' ] jq -e '.ready == true and .service == "video-factory"' /tmp/video-factory-ready.json >/dev/null + evidence_tmp="$(mktemp)" + jq '. + { + work_console: "ready", + work_api: "ready", + work_redirects: "public-host-safe", + video_factory_console: "ready", + video_factory_ready_endpoint: "ready" + }' /tmp/video-factory-production-ready.json > "${evidence_tmp}" + mv "${evidence_tmp}" /tmp/video-factory-production-ready.json + - name: Remove administrator session material if: always() - run: rm -f /tmp/admin-cookie.txt + run: | + rm -f \ + /tmp/admin-cookie.txt \ + /tmp/work-session.cookies - name: Upload sanitized verification evidence if: always() From 19377960a026524d3dc72e44971b5bab7a7ae490 Mon Sep 17 00:00:00 2001 From: Gracecom <64633490+Gracecom1@users.noreply.github.com> Date: Fri, 31 Jul 2026 06:35:29 +0900 Subject: [PATCH 3/3] fix: use the deployed Video Factory v1 proxy --- .../workflows/verify-video-factory-production-ready.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/verify-video-factory-production-ready.yml b/.github/workflows/verify-video-factory-production-ready.yml index c3b42eb7a..e83eb90d4 100644 --- a/.github/workflows/verify-video-factory-production-ready.yml +++ b/.github/workflows/verify-video-factory-production-ready.yml @@ -70,10 +70,10 @@ jobs: exit 1 fi } - fetch '/api/video-factory/v1/console/bootstrap' /tmp/factory-bootstrap.json - fetch '/api/video-factory/v1/runtime' /tmp/factory-runtime.json - fetch '/api/video-factory/v1/registry' /tmp/factory-registry.json - fetch '/api/video-factory/v1/vast/instances' /tmp/factory-instances.json + fetch '/v1/console/bootstrap' /tmp/factory-bootstrap.json + fetch '/v1/runtime' /tmp/factory-runtime.json + fetch '/v1/registry' /tmp/factory-registry.json + fetch '/v1/vast/instances' /tmp/factory-instances.json - name: Assert real production readiness run: |