diff --git a/.github/workflows/all_test.yml b/.github/workflows/all_test.yml index 183b592..24755a6 100644 --- a/.github/workflows/all_test.yml +++ b/.github/workflows/all_test.yml @@ -65,10 +65,19 @@ jobs: # Scene selection: # - ci_top_attention_doc_page_build validates doc build through the prebuilt Docker image. # - ci_top_attention_bin_kvtest keeps the Rust kv_test entry under the testbed scene contract. + # - ci_top_attention_config_* and ci_top_attention_ctrl_c_* reuse the same ops testbed/test-stack CI chain. suite["scenes"] = { key: value for key, value in suite["scenes"].items() - if key in ("ci_top_attention_doc_page_build", "ci_top_attention_bin_kvtest") + if key in ( + "ci_top_attention_doc_page_build", + "ci_top_attention_bin_kvtest", + "ci_top_attention_config_kv", + "ci_top_attention_config_fs", + "ci_top_attention_config_mq", + "ci_top_attention_ctrl_c_kv", + "ci_top_attention_ctrl_c_mq", + ) } # Profile selection: @@ -91,6 +100,7 @@ jobs: # - Keep the original per-scene scales from ci_test_list.yaml. # - ci_top_attention_doc_page_build stays on n1_kvowner_dram_3gib. # - ci_top_attention_bin_kvtest stays on n1_kvowner_dram_20gib. + # - Config/Ctrl-C wrappers stay on n1_kvowner_dram_3gib. out_path.write_text( yaml.safe_dump(suite, sort_keys=False, allow_unicode=False), diff --git "a/fluxon_doc_cn/design/teststack_1_\345\275\223\345\211\215\346\236\266\346\236\204\344\270\216CI\346\265\213\350\257\225\346\265\201\347\250\213.md" "b/fluxon_doc_cn/design/teststack_1_\345\275\223\345\211\215\346\236\266\346\236\204\344\270\216CI\346\265\213\350\257\225\346\265\201\347\250\213.md" index ce767f7..866f925 100644 --- "a/fluxon_doc_cn/design/teststack_1_\345\275\223\345\211\215\346\236\266\346\236\204\344\270\216CI\346\265\213\350\257\225\346\265\201\347\250\213.md" +++ "b/fluxon_doc_cn/design/teststack_1_\345\275\223\345\211\215\346\236\266\346\236\204\344\270\216CI\346\265\213\350\257\225\346\265\201\347\250\213.md" @@ -247,7 +247,7 @@ suite 中有两大类场景: - 这是 case 空间的第一层边界。 2. `scene kind` - 决定当前 scene 走 `CI` 还是 `TEST_STACK` 分支; - - 也决定后续必须存在哪些字段,例如 `scene.ci.runtime_contract` 或 `scene.test_stack.mode`。 + - 也决定后续必须存在哪些字段,例如 `scene.ci.requirements` 或 `scene.test_stack.mode`。 3. `scale` - 提供 topology、targets、owner、benchmark 等“规模与布局”约束; - 这些约束不会自己决定 runtime 模板,但会限制哪些 runtime 模板能成功 materialize。 @@ -292,7 +292,7 @@ suite 中有两大类场景: - `CI` - scene 选中的 profile 必须有 `runtime.ci`; - - `scene.ci.runtime_contract` 必须能在该 profile 的 `runtime.ci.runtime_contracts` 中找到; + - `scene.ci.requirements` 中声明的每个 requirement,必须都能在该 profile 的 `runtime.ci.requirements` 中找到; - 选中的 scale 必须至少满足 `CI` 所需的 topology 基础字段。 - `TEST_STACK` - scene 选中的 profile 必须有 `runtime.test_stack`; @@ -308,7 +308,7 @@ suite 中有两大类场景: - `scene.select.profiles` 引用了不存在的 `profile`。 - `profile.artifact_set` 指向了不存在的 `artifact_set`。 - `CI` scene 选中了没有 `runtime.ci` 的 profile。 -- `scene.ci.runtime_contract` 在选中 profile 的 `runtime.ci.runtime_contracts` 中不存在。 +- `scene.ci.requirements` 中有 requirement 未在选中 profile 的 `runtime.ci.requirements` 中定义。 - `TEST_STACK` scene 选中了没有 benchmark block 的 scale。 - `TEST_STACK` role plan 推导出的 target,不在 profile 的 `deploy.target_ip_map` 里。 - `run.selectors.profile_ids` 或 `case_ids` 在编译后的 case 集合里选不中任何对象。 @@ -348,7 +348,7 @@ deploy.instances 不写死在 suite 中。Runner 会结合 scale、profile 和 `CI` scene 会在通用编译模型上追加这些字段: - `scene.ci.subject` -- `scene.ci.runtime_contract` +- `scene.ci.requirements` - `scene.ci.prepare` - `scene.ci.commands` @@ -356,12 +356,17 @@ deploy.instances 不写死在 suite 中。Runner 会结合 scale、profile 和 - `prepare` 是 CI case 的前置环境准备; - `commands` 是 resolved case 里的编译产物字段,不是 suite 输入字段;`test_runner.py` 会按有限 `scene_id` 分支把它生成给 `ci_runner` 顺序执行; -- `runtime_contract` 决定 profile 里选哪套 runtime 模板。 +- `requirements` 显式声明这个 scene 需要哪些基础服务、case runtime 实例,以及额外 runner 行为。 -已存在两个 runtime contract: +当前 requirement 是有限枚举集合,例如: -- `cluster_kv_owner` -- `rust_self_managed` +- `testbed_etcd` +- `testbed_greptime` +- `master` +- `owner_0` +- `ci_runner` +- `owner_shared_bundle` +- `fluxon_kv_readiness_probe` `_compile_ci_case()` 会根据: @@ -374,7 +379,7 @@ deploy.instances 不写死在 suite 中。Runner 会结合 scale、profile 和 `CI` 特化的稳定事实: - `CI_CASE_RUNTIME_INSTANCE_IDS = ("master", "owner_0", "ci_runner")` -- 最终是否包含这三个实例,取决于 runtime contract 模板里是否声明; +- 最终是否包含这些实例,取决于 scene requirements 是否显式声明并且 profile requirement configs 是否提供; - `resolved_case` 会额外固化 `command_id`、`test_id` 等 CI 元数据; - 生成顺序是稳定的,后续 phase 规划依赖这个顺序。 diff --git a/fluxon_test_stack/ci_2_virt_node.py b/fluxon_test_stack/ci_2_virt_node.py index 28e9b82..48a0a23 100644 --- a/fluxon_test_stack/ci_2_virt_node.py +++ b/fluxon_test_stack/ci_2_virt_node.py @@ -39,8 +39,6 @@ LOCAL_SECONDARY_NODE_SUFFIX = "b" TEST_STACK_START_TEST_BED_CONFIG_ENV = "FLUXON_TEST_STACK_START_TEST_BED_CONFIG" PLACEHOLDER_WHEEL_NAME = "fluxon-0.0.0-ci-placeholder-cp38-abi3-manylinux_2_28_x86_64.whl" -SAME_HOST_LOCAL_MULTI_NODE_ETCD_CLIENT_PORT_OFFSET = 100 -SAME_HOST_LOCAL_MULTI_NODE_GREPTIME_PORT_OFFSET = 110 def _parse_args() -> argparse.Namespace: @@ -404,10 +402,6 @@ def _rewrite_suite_for_local_dual_nodes( runtime = generated_profile.get("runtime") if not isinstance(runtime, dict): raise ValueError("generated public profile runtime must be a mapping") - ci_base_runtime_host_ports = { - "etcd": int(controller_port) + SAME_HOST_LOCAL_MULTI_NODE_ETCD_CLIENT_PORT_OFFSET, - "greptime": int(controller_port) + SAME_HOST_LOCAL_MULTI_NODE_GREPTIME_PORT_OFFSET, - } ci_runtime = runtime.get("ci") if not isinstance(ci_runtime, dict): raise ValueError("generated public profile must define runtime.ci") @@ -435,28 +429,14 @@ def _rewrite_suite_for_local_dual_nodes( secondary_node_name: host_ip, } if runtime_key == "ci": - runtime_contracts = runtime_block.get("runtime_contracts") - if not isinstance(runtime_contracts, dict): - raise ValueError("generated public profile runtime.ci.runtime_contracts must be a mapping") - for contract in runtime_contracts.values(): - if not isinstance(contract, dict): - continue - base_runtime = contract.get("base_runtime") - if isinstance(base_runtime, dict): - for svc_name in ("etcd", "greptime"): - svc_cfg = base_runtime.get(svc_name) - if isinstance(svc_cfg, dict): - svc_cfg["target"] = primary_node_name - endpoint_cfg = svc_cfg.get("endpoint") - if isinstance(endpoint_cfg, dict): - endpoint_cfg["host_port"] = int(ci_base_runtime_host_ports[svc_name]) - case_runtime = contract.get("case_runtime") - if isinstance(case_runtime, dict): - master_cfg = case_runtime.get("master") - if isinstance(master_cfg, dict): - deployer_cfg = master_cfg.get("deployer") - if isinstance(deployer_cfg, dict): - deployer_cfg["target"] = primary_node_name + requirement_configs = runtime_block.get("requirements") + if not isinstance(requirement_configs, dict): + raise ValueError("generated public profile runtime.ci.requirements must be a mapping") + master_cfg = requirement_configs.get("master") + if isinstance(master_cfg, dict): + deployer_cfg = master_cfg.get("deployer") + if isinstance(deployer_cfg, dict): + deployer_cfg["target"] = primary_node_name if runtime_key == "test_stack": deploy_templates = runtime_block.get("deploy_templates") if isinstance(deploy_templates, dict): diff --git a/fluxon_test_stack/ci_test_list.yaml b/fluxon_test_stack/ci_test_list.yaml index 482cb79..695749d 100644 --- a/fluxon_test_stack/ci_test_list.yaml +++ b/fluxon_test_stack/ci_test_list.yaml @@ -1,4 +1,4 @@ -schema_version: 9 +schema_version: 10 run: mode: full_once @@ -11,8 +11,7 @@ run: scenes: ci_top_attention_doc_page_build: ci: - subject: doc_page - runtime_contract: rust_self_managed + requirements: [ci_runner] prepare: - kind: online_docker_image image_ref: hanbaoaaa/fluxon-doc-site-builder:quartz-v5.0.0-node-v24.16.0 @@ -23,12 +22,46 @@ scenes: ci_top_attention_bin_kvtest: ci: - subject: rust - runtime_contract: rust_self_managed + requirements: [ci_runner, testbed_etcd, testbed_greptime] select: scales: [n1_kvowner_dram_20gib] profiles: [fluxon_tcp] + ci_top_attention_config_kv: + ci: + requirements: [ci_runner] + select: + scales: [n1_kvowner_dram_3gib] + profiles: [fluxon_tcp] + + ci_top_attention_config_fs: + ci: + requirements: [ci_runner] + select: + scales: [n1_kvowner_dram_3gib] + profiles: [fluxon_tcp] + + ci_top_attention_config_mq: + ci: + requirements: [ci_runner, master, owner_0, testbed_etcd, testbed_greptime] + select: + scales: [n1_kvowner_dram_3gib] + profiles: [fluxon_tcp] + + ci_top_attention_ctrl_c_kv: + ci: + requirements: [ci_runner] + select: + scales: [n1_kvowner_dram_3gib] + profiles: [fluxon_tcp] + + ci_top_attention_ctrl_c_mq: + ci: + requirements: [ci_runner, testbed_etcd, testbed_greptime] + select: + scales: [n1_kvowner_dram_3gib] + profiles: [fluxon_tcp] + kv_read_heavy_zipf: test_stack: mode: KVSTORE @@ -315,72 +348,48 @@ profiles: doc_site_base_url: example.com ci_top_attention_bin_kvtest: kv_test_rounds: all - runtime_contracts: - cluster_kv_owner: &cluster_kv_owner_runtime - base_runtime: - etcd: - target: infra44-ThinkStation-PX - endpoint: - scheme: HTTP - host_port: 32579 - greptime: - target: infra44-ThinkStation-PX - endpoint: - scheme: HTTP - host_port: 34030 - case_runtime: - master: - lifecycle: service - k8s_ref: deployment/master - deployer: - target: infra44-ThinkStation-PX - command: [/bin/bash, -lc] - args: - - | - set -euo pipefail - cd __RUN_DIR__/src - exec __RUN_DIR__/venv/bin/python3 -m fluxon_py.runtime.start_master \ - -c __RUN_DIR__/configs/ci_master.yaml \ - -w __RUN_DIR__/services/master - owner_0: - lifecycle: service - k8s_ref: deployment/owner_0 - deployer: - target: __TARGET__ - command: [/bin/bash, -lc] - args: - - | - set -euo pipefail - cd __RUN_DIR__/src - exec __RUN_DIR__/venv/bin/python3 -m fluxon_py.runtime.start_owner_kvclient \ - -c __RUN_DIR__/configs/ci_owner_0.yaml \ - -w __RUN_DIR__/services/owner_0 - ci_runner: &common_ci_runner_instance - lifecycle: job - k8s_ref: deployment/ci_runner - deployer: - target: __TARGET__ - command: [/bin/bash, -lc] - args: - - | - set -uo pipefail - exec bash __RUN_DIR__/ci_runner.sh - rust_self_managed: - base_runtime: - etcd: - target: infra44-ThinkStation-PX - endpoint: - scheme: HTTP - host_port: 32579 - greptime: - target: infra44-ThinkStation-PX - endpoint: - scheme: HTTP - host_port: 34030 - case_runtime: - ci_runner: - <<: *common_ci_runner_instance - + ci_top_attention_config_kv: {} + ci_top_attention_config_fs: {} + ci_top_attention_config_mq: {} + ci_top_attention_ctrl_c_kv: {} + ci_top_attention_ctrl_c_mq: {} + requirements: + master: + lifecycle: service + k8s_ref: deployment/master + deployer: + target: infra44-ThinkStation-PX + command: [/bin/bash, -lc] + args: + - | + set -euo pipefail + cd __RUN_DIR__/src + exec __RUN_DIR__/venv/bin/python3 -m fluxon_py.runtime.start_master \ + -c __RUN_DIR__/configs/ci_master.yaml \ + -w __RUN_DIR__/services/master + owner_0: + lifecycle: service + k8s_ref: deployment/owner_0 + deployer: + target: __TARGET__ + command: [/bin/bash, -lc] + args: + - | + set -euo pipefail + cd __RUN_DIR__/src + exec __RUN_DIR__/venv/bin/python3 -m fluxon_py.runtime.start_owner_kvclient \ + -c __RUN_DIR__/configs/ci_owner_0.yaml \ + -w __RUN_DIR__/services/owner_0 + ci_runner: &common_ci_runner_instance + lifecycle: job + k8s_ref: deployment/ci_runner + deployer: + target: __TARGET__ + command: [/bin/bash, -lc] + args: + - | + set -uo pipefail + exec bash __RUN_DIR__/ci_runner.sh test_stack: &common_test_stack_runtime kind: FLUXON # English note: @@ -460,6 +469,11 @@ profiles: doc_site_base_url: example.com ci_top_attention_bin_kvtest: kv_test_rounds: all + ci_top_attention_config_kv: {} + ci_top_attention_config_fs: {} + ci_top_attention_config_mq: {} + ci_top_attention_ctrl_c_kv: {} + ci_top_attention_ctrl_c_mq: {} test_stack: <<: *common_test_stack_runtime fluxon_sockudo_ws: @@ -472,6 +486,11 @@ profiles: doc_site_base_url: example.com ci_top_attention_bin_kvtest: kv_test_rounds: all + ci_top_attention_config_kv: {} + ci_top_attention_config_fs: {} + ci_top_attention_config_mq: {} + ci_top_attention_ctrl_c_kv: {} + ci_top_attention_ctrl_c_mq: {} test_stack: <<: *common_test_stack_runtime fluxon_tcp: @@ -484,6 +503,11 @@ profiles: doc_site_base_url: example.com ci_top_attention_bin_kvtest: kv_test_rounds: all + ci_top_attention_config_kv: {} + ci_top_attention_config_fs: {} + ci_top_attention_config_mq: {} + ci_top_attention_ctrl_c_kv: {} + ci_top_attention_ctrl_c_mq: {} test_stack: <<: *common_test_stack_runtime redis_sharded: diff --git a/fluxon_test_stack/test_runner.py b/fluxon_test_stack/test_runner.py index f70618a..233b999 100644 --- a/fluxon_test_stack/test_runner.py +++ b/fluxon_test_stack/test_runner.py @@ -59,7 +59,7 @@ # # Keep them decoupled to avoid accidental "schema bumps" across unrelated layers. SCHEMA_VERSION = 1 -SUITE_SCHEMA_VERSION = 9 +SUITE_SCHEMA_VERSION = 10 # Enums (case-sensitive strings; internal routing only - not part of suite config schema) SCENE_KIND_INFER = "INFER" @@ -78,12 +78,6 @@ _RUN_EXCEPTION_FILENAME = "exception.txt" CI_PRESERVED_APPLY_IDS_SCHEMA_VERSION = 1 CI_PRESERVED_APPLY_IDS_FILENAME = "ci_preserved_apply_ids.yaml" -CI_RUNTIME_CONTRACT_CLUSTER_KV_OWNER = "cluster_kv_owner" -CI_RUNTIME_CONTRACT_RUST_SELF_MANAGED = "rust_self_managed" -CI_RUNTIME_CONTRACT_IDS = ( - CI_RUNTIME_CONTRACT_CLUSTER_KV_OWNER, - CI_RUNTIME_CONTRACT_RUST_SELF_MANAGED, -) CI_PREPARE_KIND_SETUP_DEV_ENV = "setup_dev_env" CI_PREPARE_KIND_ONLINE_DOCKER_IMAGE = "online_docker_image" CI_PREPARE_KIND_IDS = ( @@ -98,7 +92,28 @@ RUNTIME_LAYER_BASE, RUNTIME_LAYER_CASE, ) +CI_REQUIREMENT_TESTBED_ETCD = "testbed_etcd" +CI_REQUIREMENT_TESTBED_GREPTIME = "testbed_greptime" +CI_REQUIREMENT_MASTER = "master" +CI_REQUIREMENT_OWNER_0 = "owner_0" +CI_REQUIREMENT_CI_RUNNER = "ci_runner" +CI_REQUIREMENT_IDS = ( + CI_REQUIREMENT_TESTBED_ETCD, + CI_REQUIREMENT_TESTBED_GREPTIME, + CI_REQUIREMENT_MASTER, + CI_REQUIREMENT_OWNER_0, + CI_REQUIREMENT_CI_RUNNER, +) CI_BASE_RUNTIME_SERVICE_IDS = ("etcd", "greptime") +CI_BASE_RUNTIME_REQUIREMENT_IDS = { + "etcd": CI_REQUIREMENT_TESTBED_ETCD, + "greptime": CI_REQUIREMENT_TESTBED_GREPTIME, +} +CI_REQUIREMENTS_WITH_PROFILE_CONFIG = ( + CI_REQUIREMENT_MASTER, + CI_REQUIREMENT_OWNER_0, + CI_REQUIREMENT_CI_RUNNER, +) CI_CLUSTER_MEMBER_INSTANCE_IDS = ("master", "owner_0") CI_CLUSTER_RUNTIME_INSTANCE_IDS = ("master", "owner_0") CI_CASE_RUNTIME_INSTANCE_IDS = ("master", "owner_0", "ci_runner") @@ -312,21 +327,6 @@ def _test_stack_ops_namespace() -> str: ) return raw -# Suite schema keeps scene as purpose+subject and pushes concrete sizing/topology into scale. -SCENE_SUBJECT_KV = "kv" -SCENE_SUBJECT_MQ = "mq" -SCENE_SUBJECT_FS = "fs" -SCENE_SUBJECT_RUST = "rust" -SCENE_SUBJECT_DOC_PAGE = "doc_page" -SCENE_SUBJECT_INFER = "infer" -SCENE_SUBJECTS_ALLOWED = { - SCENE_SUBJECT_KV, - SCENE_SUBJECT_MQ, - SCENE_SUBJECT_FS, - SCENE_SUBJECT_RUST, - SCENE_SUBJECT_DOC_PAGE, - SCENE_SUBJECT_INFER, -} TEST_STACK_REQUEST_DISTRIBUTION_UNIFORM = "uniform" TEST_STACK_REQUEST_DISTRIBUTION_ZIPFIAN = "zipfian" TEST_STACK_REQUEST_DISTRIBUTIONS_ALLOWED = { @@ -356,6 +356,11 @@ def _runner_native_ci_scene_ids() -> Tuple[str, ...]: return ( "ci_top_attention_doc_page_build", "ci_top_attention_bin_kvtest", + "ci_top_attention_config_kv", + "ci_top_attention_config_fs", + "ci_top_attention_config_mq", + "ci_top_attention_ctrl_c_kv", + "ci_top_attention_ctrl_c_mq", ) @@ -2580,15 +2585,30 @@ def _ci_has_instance(resolved_case: Dict[str, Any], *, instance_id: str) -> bool return instance_id in set(_ci_case_instance_ids(resolved_case)) -def _ci_runtime_contract_id(resolved_case: Dict[str, Any]) -> str: +def _ci_requirement_ids(resolved_case: Dict[str, Any]) -> Tuple[str, ...]: scene = _require_dict(resolved_case.get("scene"), "resolved_case.scene") ci = _require_dict(scene.get("ci"), "resolved_case.scene.ci") - return _require_ci_runtime_contract( - ci.get("runtime_contract"), - "resolved_case.scene.ci.runtime_contract", + return tuple( + _parse_ci_requirements( + ci.get("requirements"), + "resolved_case.scene.ci.requirements", + ) ) +def _ci_has_requirement(resolved_case: Dict[str, Any], *, requirement_id: str) -> bool: + return requirement_id in set(_ci_requirement_ids(resolved_case)) + + +def _selected_ci_requirement_configs(resolved_case: Dict[str, Any]) -> Dict[str, Any]: + profile = _require_dict(resolved_case.get("profile"), "resolved_case.profile") + ci = _require_dict(profile.get("ci"), "resolved_case.profile.ci") + raw_requirement_configs = ci.get("requirements") + if raw_requirement_configs is None: + return {} + return _require_dict(raw_requirement_configs, "resolved_case.profile.ci.requirements") + + def _case_family_id(case_kind: str) -> str: if case_kind == SCENE_KIND_INFER: return CASE_FAMILY_INFER @@ -2630,9 +2650,14 @@ def _close_case_runtime_locks(runtime_tracking: _CaseRuntimeTracking) -> None: runtime_tracking.controller_lock_fp = None -def _build_runtime_model(case_family: str) -> Dict[str, Any]: +def _build_runtime_model(case_family: str, *, ci_requirement_ids: Optional[Tuple[str, ...]] = None) -> Dict[str, Any]: if case_family == CASE_FAMILY_CI: - case_instance_ids = list(CI_RUNTIME_LAYER_INSTANCE_IDS[RUNTIME_LAYER_CASE]) + requirement_ids = set(ci_requirement_ids or ()) + case_instance_ids = [ + instance_id + for instance_id in CI_CASE_RUNTIME_INSTANCE_IDS + if instance_id in requirement_ids + ] elif case_family in (CASE_FAMILY_INFER, CASE_FAMILY_BENCH): case_instance_ids = [] else: @@ -2643,7 +2668,12 @@ def _build_runtime_model(case_family: str) -> Dict[str, Any]: RUNTIME_LAYER_CASE: {"instance_ids": case_instance_ids}, } if case_family == CASE_FAMILY_CI: - model[RUNTIME_LAYER_BASE]["service_ids"] = list(CI_BASE_RUNTIME_SERVICE_IDS) + requirement_ids = set(ci_requirement_ids or ()) + model[RUNTIME_LAYER_BASE]["service_ids"] = [ + service_id + for service_id in CI_BASE_RUNTIME_SERVICE_IDS + if CI_BASE_RUNTIME_REQUIREMENT_IDS[service_id] in requirement_ids + ] return model @@ -3302,7 +3332,7 @@ def _ci_runner_runtime_stage(resolved_case: Dict[str, Any]) -> _RemoteRunDirStag if _ci_has_instance(resolved_case, instance_id="master"): verify_relpaths.append("configs/ci_master.yaml") include_relpaths = list(CI_RUNNER_REMOTE_STAGE_INCLUDE_RELPATHS) - if _ci_runtime_contract_id(resolved_case) == CI_RUNTIME_CONTRACT_CLUSTER_KV_OWNER: + if _ci_has_requirement(resolved_case, requirement_id=CI_REQUIREMENT_MASTER): for relpath in ("fluxon_release", "test_rsc"): if relpath not in include_relpaths: include_relpaths.append(relpath) @@ -3667,9 +3697,12 @@ def _prepare_ci_case( if prepare_env_exports: _write_ci_prepare_env_script(run_dir=run_dir, exports=prepare_env_exports) - profile = _require_dict(resolved_case.get("profile"), "resolved_case.profile") - profile_ci = _require_dict(profile.get("ci"), "resolved_case.profile.ci") - if profile_ci.get("scene_config") is not None: + if _scene_id_uses_runner_native_ci_commands( + _require_str( + _require_dict(resolved_case.get("case"), "resolved_case.case").get("scene_id"), + "resolved_case.case.scene_id", + ) + ): _write_ci_scene_config_yaml( resolved_case, run_dir=run_dir, @@ -4331,8 +4364,8 @@ def _ci_base_runtime_service(resolved_case: Dict[str, Any], *, service_id: str) def _ci_base_runtime_service_target_name(resolved_case: Dict[str, Any], *, service_id: str) -> str: - svc = _ci_base_runtime_service(resolved_case, service_id=service_id) - return _require_str(svc.get("target"), f"resolved_case.profile.ci.runtime.base_runtime[{service_id!r}].target") + _ = resolved_case + return _testbed_service_target_name(service_id) def _target_uses_local_loopback( @@ -4359,28 +4392,13 @@ def _ci_base_runtime_service_target_ip(resolved_case: Dict[str, Any], *, service def _ci_base_runtime_service_port(resolved_case: Dict[str, Any], *, service_id: str) -> int: - svc = _ci_base_runtime_service(resolved_case, service_id=service_id) - endpoint = _require_dict( - svc.get("endpoint"), - f"resolved_case.profile.ci.runtime.base_runtime[{service_id!r}].endpoint", - ) - return _require_int( - endpoint.get("host_port"), - f"resolved_case.profile.ci.runtime.base_runtime[{service_id!r}].endpoint.host_port", - min_v=1, - ) + _, port = _testbed_service_host_port(service_id, ctx=f"CI {service_id} service") + return port def _ci_base_runtime_service_url(resolved_case: Dict[str, Any], *, service_id: str) -> str: - svc = _ci_base_runtime_service(resolved_case, service_id=service_id) - endpoint = _require_dict( - svc.get("endpoint"), - f"resolved_case.profile.ci.runtime.base_runtime[{service_id!r}].endpoint", - ) - scheme = _require_str( - endpoint.get("scheme"), - f"resolved_case.profile.ci.runtime.base_runtime[{service_id!r}].endpoint.scheme", - ) + _ = resolved_case + scheme = _ENDPOINT_SCHEME_HTTP host = _ci_base_runtime_service_target_ip(resolved_case, service_id=service_id) port = _ci_base_runtime_service_port(resolved_case, service_id=service_id) if scheme == _ENDPOINT_SCHEME_HTTP: @@ -5651,23 +5669,6 @@ def _require_id_list(raw: Any, ctx: str) -> List[str]: return out -def _require_scene_subject(raw: Any, ctx: str) -> str: - subject = _require_str(raw, ctx).strip() - if subject not in SCENE_SUBJECTS_ALLOWED: - raise ValueError(f"{ctx} invalid subject: {subject!r}") - return subject - - -def _infer_test_stack_subject_from_mode(mode: str) -> str: - if mode == TEST_STACK_MODE_MPMC: - return SCENE_SUBJECT_MQ - if mode in (TEST_STACK_MODE_KVSTORE, TEST_STACK_MODE_KVSTORE_WITH_LOCAL_CACHE, TEST_STACK_MODE_RPC): - return SCENE_SUBJECT_KV - if mode == TEST_STACK_MODE_PY_FS: - return SCENE_SUBJECT_FS - raise ValueError(f"unsupported test_stack mode for subject inference: {mode!r}") - - def _require_test_stack_backend_kind(raw: Any, ctx: str) -> str: if raw is None: return TEST_STACK_BACKEND_FLUXON @@ -5679,23 +5680,37 @@ def _require_test_stack_backend_kind(raw: Any, ctx: str) -> str: return kind -def _test_stack_backend_supports_subject(*, backend_kind: str, subject: str) -> bool: +def _test_stack_backend_supports_mode(*, backend_kind: str, mode: str) -> bool: if backend_kind == TEST_STACK_BACKEND_FLUXON: - return subject in {SCENE_SUBJECT_KV, SCENE_SUBJECT_MQ, SCENE_SUBJECT_FS} + return mode in ( + TEST_STACK_MODE_MPMC, + TEST_STACK_MODE_KVSTORE, + TEST_STACK_MODE_KVSTORE_WITH_LOCAL_CACHE, + TEST_STACK_MODE_PY_FS, + TEST_STACK_MODE_RPC, + ) if backend_kind == TEST_STACK_BACKEND_REDIS: - return subject == SCENE_SUBJECT_KV + return mode in ( + TEST_STACK_MODE_KVSTORE, + TEST_STACK_MODE_KVSTORE_WITH_LOCAL_CACHE, + TEST_STACK_MODE_RPC, + ) if backend_kind == TEST_STACK_BACKEND_MOONCAKE: - return subject == SCENE_SUBJECT_KV + return mode in ( + TEST_STACK_MODE_KVSTORE, + TEST_STACK_MODE_KVSTORE_WITH_LOCAL_CACHE, + TEST_STACK_MODE_RPC, + ) if backend_kind == TEST_STACK_BACKEND_ALLUXIO: - return subject == SCENE_SUBJECT_FS + return mode == TEST_STACK_MODE_PY_FS raise ValueError(f"unsupported test_stack backend_kind: {backend_kind!r}") -def _validate_test_stack_backend_subject(*, backend_kind: str, subject: str, ctx: str) -> None: - if _test_stack_backend_supports_subject(backend_kind=backend_kind, subject=subject): +def _validate_test_stack_backend_mode(*, backend_kind: str, mode: str, ctx: str) -> None: + if _test_stack_backend_supports_mode(backend_kind=backend_kind, mode=mode): return raise ValueError( - f"{ctx} backend_kind={backend_kind!r} does not support subject={subject!r}" + f"{ctx} backend_kind={backend_kind!r} does not support mode={mode!r}" ) @@ -5729,27 +5744,30 @@ def _test_stack_backend_uses_dedicated_kv_owners(*, backend_kind: str, mode: str return False -def _require_ci_runtime_contract(raw: Any, ctx: str) -> str: - runtime_contract = _require_str(raw, ctx).strip() - if runtime_contract not in CI_RUNTIME_CONTRACT_IDS: - raise ValueError(f"{ctx} invalid ci runtime_contract: {runtime_contract!r}") - return runtime_contract +def _require_ci_requirement_id(raw: Any, ctx: str) -> str: + requirement_id = _require_str(raw, ctx).strip() + if requirement_id not in CI_REQUIREMENT_IDS: + raise ValueError(f"{ctx} invalid ci requirement: {requirement_id!r}") + return requirement_id -def _validate_test_stack_subject_mode(*, subject: str, mode: str, ctx: str) -> None: - if subject == SCENE_SUBJECT_MQ: - if mode != TEST_STACK_MODE_MPMC: - raise ValueError(f"{ctx} subject={subject!r} requires mode={TEST_STACK_MODE_MPMC!r}") - return - if subject == SCENE_SUBJECT_KV: - if mode not in (TEST_STACK_MODE_KVSTORE, TEST_STACK_MODE_KVSTORE_WITH_LOCAL_CACHE, TEST_STACK_MODE_RPC): - raise ValueError(f"{ctx} subject={subject!r} requires a KV-family mode") - return - if subject == SCENE_SUBJECT_FS: - if mode != TEST_STACK_MODE_PY_FS: - raise ValueError(f"{ctx} subject={subject!r} requires mode={TEST_STACK_MODE_PY_FS!r}") - return - raise ValueError(f"{ctx} unsupported test_stack subject: {subject!r}") +def _parse_ci_requirements(raw_requirements: Any, ctx: str) -> List[str]: + requirements = _require_list(raw_requirements, ctx) + if not requirements: + raise ValueError(f"{ctx} must be non-empty") + out: List[str] = [] + seen: set[str] = set() + for index, raw_requirement in enumerate(requirements): + requirement_id = _require_ci_requirement_id(raw_requirement, f"{ctx}[{index}]") + if requirement_id in seen: + raise ValueError(f"{ctx} contains duplicate requirement: {requirement_id!r}") + seen.add(requirement_id) + out.append(requirement_id) + if CI_REQUIREMENT_CI_RUNNER not in seen: + raise ValueError(f"{ctx} must include {CI_REQUIREMENT_CI_RUNNER!r}") + if CI_REQUIREMENT_OWNER_0 in seen and CI_REQUIREMENT_MASTER not in seen: + raise ValueError(f"{ctx} cannot include {CI_REQUIREMENT_OWNER_0!r} without {CI_REQUIREMENT_MASTER!r}") + return out def _parse_scene_value_size_weighted_set(raw_val: Any, *, ctx: str) -> List[Dict[str, Any]]: @@ -6240,15 +6258,11 @@ def _parse_scene(item: Dict[str, Any], ctx: str) -> Dict[str, Any]: if kind == SCENE_KIND_CI: _forbid_unknown_keys(item, {"ci", "select"}, ctx) ci = _require_dict(item.get("ci"), f"{ctx}.ci") - _forbid_unknown_keys(ci, {"subject", "runtime_contract", "prepare"}, f"{ctx}.ci") - subject = _require_scene_subject(ci.get("subject"), f"{ctx}.ci.subject") - if subject == SCENE_SUBJECT_INFER: - raise ValueError(f"{ctx}.ci.subject must not be {SCENE_SUBJECT_INFER!r}") + _forbid_unknown_keys(ci, {"requirements", "prepare"}, f"{ctx}.ci") parsed_ci = { - "subject": subject, - "runtime_contract": _require_ci_runtime_contract( - ci.get("runtime_contract"), - f"{ctx}.ci.runtime_contract", + "requirements": _parse_ci_requirements( + ci.get("requirements"), + f"{ctx}.ci.requirements", ), } raw_prepare = ci.get("prepare") @@ -6262,7 +6276,6 @@ def _parse_scene(item: Dict[str, Any], ctx: str) -> Dict[str, Any]: _forbid_unknown_keys( ts, { - "subject", "mode", "role_weights", "read_ratio", @@ -6281,13 +6294,7 @@ def _parse_scene(item: Dict[str, Any], ctx: str) -> Dict[str, Any]: f"{ctx}.test_stack", ) mode = _require_str(ts.get("mode"), f"{ctx}.test_stack.mode") - subject_raw = ts.get("subject") - subject = _infer_test_stack_subject_from_mode(mode) - if subject_raw is not None: - subject = _require_scene_subject(subject_raw, f"{ctx}.test_stack.subject") - _validate_test_stack_subject_mode(subject=subject, mode=mode, ctx=f"{ctx}.test_stack") - - out_ts: Dict[str, Any] = {"subject": subject, "mode": mode} + out_ts: Dict[str, Any] = {"mode": mode} role_weights = ts.get("role_weights") if mode == TEST_STACK_MODE_MPMC: rw = _require_dict(role_weights, f"{ctx}.test_stack.role_weights") @@ -6841,6 +6848,36 @@ def _validate_profile_ci_runtime_block(runtime: Dict[str, Any], ctx: str, target raise ValueError(f"{tpl_ctx}.endpoint must be omitted") +def _validate_profile_ci_requirement_configs( + requirement_configs: Dict[str, Any], + *, + ctx: str, + target_ip_map: Dict[str, Any], +) -> None: + _forbid_unknown_keys(requirement_configs, set(CI_REQUIREMENTS_WITH_PROFILE_CONFIG), ctx) + + case_runtime: Dict[str, Any] = {} + for instance_id in (CI_REQUIREMENT_MASTER, CI_REQUIREMENT_OWNER_0, CI_REQUIREMENT_CI_RUNNER): + raw_cfg = requirement_configs.get(instance_id) + if raw_cfg is None: + continue + case_runtime[instance_id] = _require_dict(raw_cfg, f"{ctx}[{instance_id!r}]") + if case_runtime: + _validate_profile_ci_runtime_block( + { + RUNTIME_LAYER_BASE: { + service_id: { + "target": next(iter(target_ip_map.keys())), + "endpoint": {"scheme": _ENDPOINT_SCHEME_HTTP, "host_port": 1}, + } + for service_id in CI_BASE_RUNTIME_SERVICE_IDS + }, + RUNTIME_LAYER_CASE: case_runtime, + }, + f"{ctx}.__case_runtime_validation__", + target_ip_map, + ) + def _require_clean_relpath(raw: Any, ctx: str) -> str: relpath = _require_str(raw, ctx).strip() if not relpath: @@ -7270,7 +7307,7 @@ def _parse_profile(item: Dict[str, Any], ctx: str) -> Dict[str, Any]: if runtime.get("ci") is not None: ci = _require_dict(runtime.get("ci"), f"{ctx}.runtime.ci") - _forbid_unknown_keys(ci, {"deploy", "runtime_contracts", "scene_configs"}, f"{ctx}.runtime.ci") + _forbid_unknown_keys(ci, {"deploy", "requirements", "scene_configs"}, f"{ctx}.runtime.ci") deploy = _require_dict(ci.get("deploy"), f"{ctx}.runtime.ci.deploy") _validate_profile_deploy_block( deploy, @@ -7279,16 +7316,16 @@ def _parse_profile(item: Dict[str, Any], ctx: str) -> Dict[str, Any]: allow_target_tokens=False, ) target_ip_map = _require_dict(deploy.get("target_ip_map"), f"{ctx}.runtime.ci.deploy.target_ip_map") - runtime_contracts = _require_dict(ci.get("runtime_contracts"), f"{ctx}.runtime.ci.runtime_contracts") - if not runtime_contracts: - raise ValueError(f"{ctx}.runtime.ci.runtime_contracts must be non-empty") - for contract_id, raw_runtime in runtime_contracts.items(): - _ = _require_ci_runtime_contract(contract_id, f"{ctx}.runtime.ci.runtime_contracts key") - _validate_profile_ci_runtime_block( - _require_dict(raw_runtime, f"{ctx}.runtime.ci.runtime_contracts[{contract_id!r}]"), - f"{ctx}.runtime.ci.runtime_contracts[{contract_id!r}]", - target_ip_map, - ) + raw_requirement_configs = ci.get("requirements") + if raw_requirement_configs is None: + requirement_configs = {} + else: + requirement_configs = _require_dict(raw_requirement_configs, f"{ctx}.runtime.ci.requirements") + _validate_profile_ci_requirement_configs( + requirement_configs, + ctx=f"{ctx}.runtime.ci.requirements", + target_ip_map=target_ip_map, + ) scene_configs = ci.get("scene_configs") if scene_configs is not None: scene_configs = _require_dict(scene_configs, f"{ctx}.runtime.ci.scene_configs") @@ -7441,24 +7478,44 @@ def _expand_cases(suite: _Suite) -> List[_ResolvedCase]: if profile_runtime.get("ci") is None: raise ValueError(f"scene[{scene_id}] kind={scene_kind} selects profile[{profile_id}] without ci block") scene_ci = _require_dict(scene.get("ci"), f"scene[{scene_id}].ci") - runtime_contract = _require_ci_runtime_contract( - scene_ci.get("runtime_contract"), - f"scene[{scene_id}].ci.runtime_contract", + requirement_ids = _parse_ci_requirements( + scene_ci.get("requirements"), + f"scene[{scene_id}].ci.requirements", ) profile_ci = _require_dict(profile_runtime.get("ci"), f"profile[{profile_id}].runtime.ci") - runtime_contracts = _require_dict( - profile_ci.get("runtime_contracts"), - f"profile[{profile_id}].runtime.ci.runtime_contracts", - ) - if runtime_contract not in runtime_contracts: + raw_requirement_configs = profile_ci.get("requirements") + if raw_requirement_configs is None: + requirement_configs = {} + else: + requirement_configs = _require_dict( + raw_requirement_configs, + f"profile[{profile_id}].runtime.ci.requirements", + ) + missing_requirements = [ + requirement_id + for requirement_id in requirement_ids + if requirement_id in CI_REQUIREMENTS_WITH_PROFILE_CONFIG and requirement_id not in requirement_configs + ] + if missing_requirements: raise ValueError( - f"scene[{scene_id}] runtime_contract={runtime_contract!r} is missing from profile[{profile_id}].runtime.ci.runtime_contracts" + f"scene[{scene_id}] requirements {missing_requirements!r} are missing from " + f"profile[{profile_id}].runtime.ci.requirements" ) elif scene_kind == SCENE_KIND_TEST_STACK: profile_ts = _require_dict(profile_runtime.get("test_stack"), f"profile[{profile_id}].runtime.test_stack") + scene_ts = _require_dict(scene.get("test_stack"), f"scene[{scene_id}].test_stack") + mode = _require_str(scene_ts.get("mode"), f"scene[{scene_id}].test_stack.mode") + backend_kind = _require_test_stack_backend_kind( + profile_ts.get("kind"), + f"profile[{profile_id}].runtime.test_stack.kind", + ) + _validate_test_stack_backend_mode( + backend_kind=backend_kind, + mode=mode, + ctx=f"profile[{profile_id}].runtime.test_stack.kind", + ) deploy = _require_dict(profile_ts.get("deploy"), f"profile[{profile_id}].runtime.test_stack.deploy") target_ip_map = _require_dict(deploy.get("target_ip_map"), f"profile[{profile_id}].runtime.test_stack.deploy.target_ip_map") - scene_ts = _require_dict(scene.get("test_stack"), f"scene[{scene_id}].test_stack") role_plan = _build_test_stack_role_plan( scene_ts, scale, @@ -7699,6 +7756,66 @@ def _runner_native_ci_commands_for_case(case: _ResolvedCase, *, ctx: str) -> Lis "timeout_seconds": 21600, } ] + if scene_id == "ci_top_attention_config_kv": + return [ + { + "id": "top_attention_config_kv", + "command": ( + "__RUN_DIR__/venv/bin/python3 -u " + "__RUN_DIR__/src/fluxon_test_stack/top_attention_test_index/_config_kv.py " + "--case-config __RUN_DIR__/configs/ci_scene_config.yaml" + ), + "timeout_seconds": 3600, + } + ] + if scene_id == "ci_top_attention_config_fs": + return [ + { + "id": "top_attention_config_fs", + "command": ( + "__RUN_DIR__/venv/bin/python3 -u " + "__RUN_DIR__/src/fluxon_test_stack/top_attention_test_index/_config_fs.py " + "--case-config __RUN_DIR__/configs/ci_scene_config.yaml" + ), + "timeout_seconds": 3600, + } + ] + if scene_id == "ci_top_attention_config_mq": + return [ + { + "id": "top_attention_config_mq", + "command": ( + "__RUN_DIR__/venv/bin/python3 -u " + "__RUN_DIR__/src/fluxon_test_stack/top_attention_test_index/_config_mq.py " + "--case-config __RUN_DIR__/configs/ci_scene_config.yaml" + ), + "timeout_seconds": 7200, + } + ] + if scene_id == "ci_top_attention_ctrl_c_kv": + return [ + { + "id": "top_attention_ctrl_c_kv", + "command": ( + "__RUN_DIR__/venv/bin/python3 -u " + "__RUN_DIR__/src/fluxon_test_stack/top_attention_test_index/_ctrl_c_kv.py " + "--case-config __RUN_DIR__/configs/ci_scene_config.yaml" + ), + "timeout_seconds": 3600, + } + ] + if scene_id == "ci_top_attention_ctrl_c_mq": + return [ + { + "id": "top_attention_ctrl_c_mq", + "command": ( + "__RUN_DIR__/venv/bin/python3 -u " + "__RUN_DIR__/src/fluxon_test_stack/top_attention_test_index/_ctrl_c_mq.py " + "--case-config __RUN_DIR__/configs/ci_scene_config.yaml" + ), + "timeout_seconds": 7200, + } + ] raise ValueError(f"{ctx} unsupported runner-native CI scene: {scene_id!r}") @@ -8513,9 +8630,9 @@ def _build_resolved_case_yaml( raise ValueError(f"ci_commands must be provided for CI case: {case.case_id}") scene_ci = _require_dict(scene_src.get("ci"), "resolved_case.scene_source.ci") - runtime_contract = _require_ci_runtime_contract( - scene_ci.get("runtime_contract"), - "resolved_case.scene_source.ci.runtime_contract", + requirement_ids = _parse_ci_requirements( + scene_ci.get("requirements"), + "resolved_case.scene_source.ci.requirements", ) topology = _require_test_stack_machine_count(scale_src.get("topology"), "resolved_case.scale_source.topology") targets = copy.deepcopy(_require_dict(scale_src.get("targets"), "resolved_case.scale_source.targets")) @@ -8530,16 +8647,32 @@ def _build_resolved_case_yaml( deploy_out = copy.deepcopy(_require_dict(profile_ci.get("deploy"), "resolved_case.profile_source.ci.deploy")) deploy_out["release_root"] = materialized_release_root deploy_out["test_rsc_root"] = materialized_test_rsc_root - runtime_contracts = _require_dict( - profile_ci.get("runtime_contracts"), - "resolved_case.profile_source.ci.runtime_contracts", - ) - selected_runtime = copy.deepcopy( - _require_dict( - runtime_contracts.get(runtime_contract), - f"resolved_case.profile_source.ci.runtime_contracts[{runtime_contract!r}]", + raw_requirement_configs = profile_ci.get("requirements") + if raw_requirement_configs is None: + requirement_configs = {} + else: + requirement_configs = _require_dict( + raw_requirement_configs, + "resolved_case.profile_source.ci.requirements", ) - ) + selected_requirement_configs: Dict[str, Any] = {} + selected_base_runtime: Dict[str, Any] = {} + selected_case_runtime: Dict[str, Any] = {} + for requirement_id in requirement_ids: + for service_id, service_requirement_id in CI_BASE_RUNTIME_REQUIREMENT_IDS.items(): + if requirement_id == service_requirement_id: + selected_base_runtime[service_id] = {} + if requirement_id not in CI_REQUIREMENTS_WITH_PROFILE_CONFIG: + continue + requirement_cfg = copy.deepcopy( + _require_dict( + requirement_configs.get(requirement_id), + f"resolved_case.profile_source.ci.requirements[{requirement_id!r}]", + ) + ) + selected_requirement_configs[requirement_id] = requirement_cfg + if requirement_id in CI_CASE_RUNTIME_INSTANCE_IDS: + selected_case_runtime[requirement_id] = requirement_cfg scene_configs = profile_ci.get("scene_configs") selected_scene_config = None if scene_configs is not None: @@ -8555,8 +8688,7 @@ def _build_resolved_case_yaml( scene = { "ci": { - "subject": _require_scene_subject(scene_ci.get("subject"), "resolved_case.scene_source.ci.subject"), - "runtime_contract": runtime_contract, + "requirements": list(requirement_ids), "commands": copy.deepcopy(ci_commands), } } @@ -8566,8 +8698,11 @@ def _build_resolved_case_yaml( profile = { "deploy": deploy_out, "ci": { - "runtime_contract": runtime_contract, - "runtime": selected_runtime, + "requirements": selected_requirement_configs, + "runtime": { + RUNTIME_LAYER_BASE: selected_base_runtime, + RUNTIME_LAYER_CASE: selected_case_runtime, + }, }, } if selected_scene_config is not None: @@ -8578,16 +8713,15 @@ def _build_resolved_case_yaml( duration_seconds = _require_int(scale_src.get("duration_seconds"), "scale.duration_seconds", min_v=1) topology = copy.deepcopy(scale_src.get("topology")) mode = _require_str(scene_ts.get("mode"), "scene.test_stack.mode") - subject = _require_scene_subject(scene_ts.get("subject"), "scene.test_stack.subject") profile_ts = _require_dict(profile_runtime_src.get("test_stack"), "resolved_case.profile_source.runtime.test_stack") backend_kind = _require_test_stack_backend_kind( profile_ts.get("kind"), "resolved_case.profile_source.test_stack.kind", ) - _validate_test_stack_backend_subject( + _validate_test_stack_backend_mode( backend_kind=backend_kind, - subject=subject, + mode=mode, ctx="resolved_case.profile_source.test_stack.kind", ) deploy_out = copy.deepcopy(_require_dict(profile_ts.get("deploy"), "resolved_case.profile_source.test_stack.deploy")) @@ -8694,7 +8828,10 @@ def _build_resolved_case_yaml( "run_dir": run_dir, "stack_identity": stack_identity, }, - "runtime_model": _build_runtime_model(case_family), + "runtime_model": _build_runtime_model( + case_family, + ci_requirement_ids=tuple(requirement_ids) if case_family == CASE_FAMILY_CI else None, + ), "artifact_set": { "id": artifact_set_id, "release_source": artifact_release_source, @@ -9687,10 +9824,9 @@ def _compile_test_stack_case(resolved_case: Dict[str, Any], *, run_index: int) - ts_profile.get("runtime_env"), "resolved_case.profile.test_stack.runtime_env", ) - scene_subject = _require_scene_subject(ts_scene.get("subject"), "resolved_case.scene.test_stack.subject") - _validate_test_stack_backend_subject( + _validate_test_stack_backend_mode( backend_kind=backend_kind, - subject=scene_subject, + mode=scene_mode, ctx="resolved_case.profile.test_stack.kind", ) port_alloc = _require_dict(ts_profile.get("port_alloc"), "profile.test_stack.port_alloc") @@ -11315,6 +11451,34 @@ def _active_test_stack_target_ip_map(*, ctx: str) -> Dict[str, str]: return out +def _testbed_service_host_port(service_id: str, *, ctx: str) -> Tuple[str, int]: + deployconf_path = _load_test_bed_deployconf_path() + deployconf = _require_dict(_load_yaml_file(deployconf_path), f"deployconf {deployconf_path}") + services = _require_dict(deployconf.get("service"), "deployconf.service") + service = _require_dict(services.get(service_id), f"deployconf.service.{service_id}") + port = _require_int(service.get("port"), f"deployconf.service.{service_id}.port", min_v=1) + node_bind = _require_dict(service.get("node_bind"), f"deployconf.service.{service_id}.node_bind") + nodes = _require_list(node_bind.get("node"), f"deployconf.service.{service_id}.node_bind.node") + if not nodes: + raise ValueError(f"deployconf.service.{service_id}.node_bind.node must be non-empty") + hostname = _require_str(nodes[0], f"deployconf.service.{service_id}.node_bind.node[0]") + target_ip_map = _active_test_stack_target_ip_map(ctx=ctx) + ip = _require_str(target_ip_map.get(hostname), f"{ctx}.target_ip_map[{hostname!r}]") + return ip, int(port) + + +def _testbed_service_target_name(service_id: str) -> str: + deployconf_path = _load_test_bed_deployconf_path() + deployconf = _require_dict(_load_yaml_file(deployconf_path), f"deployconf {deployconf_path}") + services = _require_dict(deployconf.get("service"), "deployconf.service") + service = _require_dict(services.get(service_id), f"deployconf.service.{service_id}") + node_bind = _require_dict(service.get("node_bind"), f"deployconf.service.{service_id}.node_bind") + nodes = _require_list(node_bind.get("node"), f"deployconf.service.{service_id}.node_bind.node") + if not nodes: + raise ValueError(f"deployconf.service.{service_id}.node_bind.node must be non-empty") + return _require_str(nodes[0], f"deployconf.service.{service_id}.node_bind.node[0]") + + def _test_stack_greptime_host_port(resolved_case: Dict[str, Any]) -> Tuple[str, int]: """Infer Greptime host:port for TEST_STACK master monitoring config from the self-host deployconf.""" deployconf_path = _load_test_bed_deployconf_path() @@ -14544,7 +14708,7 @@ def _ci_cluster_member_target_ips(resolved_case: Dict[str, Any]) -> List[str]: def _resolved_ci_command_list(resolved_case: Dict[str, Any]) -> List[Dict[str, str]]: scene = _require_dict(resolved_case.get("scene"), "resolved_case.scene") ci = _require_dict(scene.get("ci"), "resolved_case.scene.ci") - _forbid_unknown_keys(ci, {"subject", "commands", "runtime_contract", "prepare"}, "resolved_case.scene.ci") + _forbid_unknown_keys(ci, {"commands", "requirements", "prepare"}, "resolved_case.scene.ci") raw_commands = _require_list(ci.get("commands"), "resolved_case.scene.ci.commands") if not raw_commands: raise ValueError("resolved_case.scene.ci.commands must be non-empty") @@ -14735,11 +14899,12 @@ def _ci_runner_exit_code_timeout_seconds(resolved_case: Dict[str, Any]) -> int: - Keep one causal source of truth: sum the explicit per-command timeouts and add the bounded pre-command phases that are also encoded in the generated script. """ - timeout_seconds = ( - CI_RUNNER_SHARED_BUNDLE_TIMEOUT_S - + CI_RUNNER_READINESS_PROBE_DEADLINE_S - + CI_RUNNER_EXIT_CODE_GRACE_TIMEOUT_S - ) + timeout_seconds = CI_RUNNER_EXIT_CODE_GRACE_TIMEOUT_S + if _ci_has_instance(resolved_case, instance_id="owner_0"): + timeout_seconds += ( + CI_RUNNER_SHARED_BUNDLE_TIMEOUT_S + + CI_RUNNER_READINESS_PROBE_DEADLINE_S + ) commands = _resolved_ci_command_list(resolved_case) for index, command in enumerate(commands): raw_timeout = command.get("timeout_seconds") @@ -14766,7 +14931,7 @@ def _write_ci_runner_script( commands = _resolved_ci_command_list(resolved_case) venv_python = run_dir / "venv" / "bin" / "python3" test_backend = (src_root / "fluxon_py" / "tests" / "test_backend.py").resolve() - requires_owner_shared_bundle = _ci_has_instance(resolved_case, instance_id="owner_0") + requires_owner_runtime = _ci_has_instance(resolved_case, instance_id="owner_0") out_path = run_dir / "ci_runner.sh" if out_path.exists(): @@ -14800,7 +14965,7 @@ def _write_ci_runner_script( shared_bundle_block = "" readiness_probe_block = "" - if requires_owner_shared_bundle: + if requires_owner_runtime: bundle_cluster_name = _ci_cluster_name(resolved_case) bundle_shared_memory_dir = str( _cluster_scoped_shared_dir(root_path=share_mem_path, cluster_name=bundle_cluster_name) @@ -14829,6 +14994,9 @@ def _write_ci_runner_script( fail_and_exit 2 fi """ + if requires_owner_runtime: + readiness_test_id = "basic_put_and_get" + readiness_instance_suffix = "readiness_probe" readiness_probe_block = f""" echo "[ci_runner] running backend readiness probe..." readiness_rc=1 @@ -14836,11 +15004,11 @@ def _write_ci_runner_script( readiness_deadline=$(( $(date +%s) + {CI_RUNNER_READINESS_PROBE_DEADLINE_S} )) while [ $(date +%s) -lt "$readiness_deadline" ]; do readiness_attempt=$((readiness_attempt + 1)) - echo "[CI readiness_probe] attempt=$readiness_attempt argv={venv_python.as_posix()} -u {test_backend.as_posix()} --test-id basic_put_and_get --instance-suffix readiness_probe" - timeout --preserve-status --signal=KILL 60 {venv_python.as_posix()} -u {test_backend.as_posix()} --test-id basic_put_and_get --instance-suffix readiness_probe + echo "[CI readiness_probe] attempt=$readiness_attempt argv={venv_python.as_posix()} -u {test_backend.as_posix()} --test-id {readiness_test_id} --instance-suffix {readiness_instance_suffix}" + timeout --preserve-status --signal=KILL 60 {venv_python.as_posix()} -u {test_backend.as_posix()} --test-id {readiness_test_id} --instance-suffix {readiness_instance_suffix} readiness_rc=$? if [ "$readiness_rc" -eq 0 ]; then - echo "[CI readiness_probe] basic_put_and_get passed on attempt=$readiness_attempt" + echo "[CI readiness_probe] {readiness_test_id} passed on attempt=$readiness_attempt" break fi echo "[CI readiness_probe] failed rc=$readiness_rc attempt=$readiness_attempt" diff --git a/fluxon_test_stack/tests/test_ci_2_virt_node_contract.py b/fluxon_test_stack/tests/test_ci_2_virt_node_contract.py index 667c00b..afaab90 100644 --- a/fluxon_test_stack/tests/test_ci_2_virt_node_contract.py +++ b/fluxon_test_stack/tests/test_ci_2_virt_node_contract.py @@ -29,6 +29,11 @@ def _load_module(): class TestCi2VirtNodeContract(unittest.TestCase): _KVTEST_SCENE_ID = "ci_top_attention_bin_kvtest" _DOC_SCENE_ID = "ci_top_attention_doc_page_build" + _CONFIG_KV_SCENE_ID = "ci_top_attention_config_kv" + _CONFIG_FS_SCENE_ID = "ci_top_attention_config_fs" + _CONFIG_MQ_SCENE_ID = "ci_top_attention_config_mq" + _CTRL_C_KV_SCENE_ID = "ci_top_attention_ctrl_c_kv" + _CTRL_C_MQ_SCENE_ID = "ci_top_attention_ctrl_c_mq" def test_generated_suite_is_public_dual_local_nodes_ci_only(self) -> None: suite_cfg = _ENTRY._load_yaml_mapping(_ENTRY.DEFAULT_SUITE_PATH, ctx="suite") @@ -55,17 +60,13 @@ def test_generated_suite_is_public_dual_local_nodes_ci_only(self) -> None: generated["profiles"]["fluxon_tcp_thread"]["runtime"]["ci"]["deploy"]["target_ip_map"], {"local-node-a": "10.1.1.119", "local-node-b": "10.1.1.119"}, ) - self.assertEqual( - generated["profiles"]["fluxon_tcp_thread"]["runtime"]["ci"]["runtime_contracts"]["cluster_kv_owner"][ - "base_runtime" - ]["etcd"]["endpoint"]["host_port"], - 19180, + self.assertNotIn( + "testbed_etcd", + generated["profiles"]["fluxon_tcp_thread"]["runtime"]["ci"]["requirements"], ) - self.assertEqual( - generated["profiles"]["fluxon_tcp_thread"]["runtime"]["ci"]["runtime_contracts"]["cluster_kv_owner"][ - "base_runtime" - ]["greptime"]["endpoint"]["host_port"], - 19190, + self.assertNotIn( + "testbed_greptime", + generated["profiles"]["fluxon_tcp_thread"]["runtime"]["ci"]["requirements"], ) self.assertEqual( generated["artifact_sets"]["fluxon_tcp_thread"]["release_source"]["key_prefix"], @@ -154,8 +155,8 @@ def test_generated_suite_supports_doc_page_ci_scene(self) -> None: self.assertEqual(set(generated["scenes"].keys()), {self._DOC_SCENE_ID}) self.assertEqual( - generated["scenes"][self._DOC_SCENE_ID]["ci"]["runtime_contract"], - "rust_self_managed", + generated["scenes"][self._DOC_SCENE_ID]["ci"]["requirements"], + ["ci_runner"], ) prepare = generated["scenes"][self._DOC_SCENE_ID]["ci"]["prepare"] self.assertEqual( @@ -175,6 +176,49 @@ def test_generated_suite_supports_doc_page_ci_scene(self) -> None: ) self.assertEqual(set(generated["scales"].keys()), {"n1_kvowner_dram_3gib"}) + def test_generated_suite_supports_top_attention_config_and_ctrl_c_ci_scenes(self) -> None: + suite_cfg = _ENTRY._load_yaml_mapping(_ENTRY.DEFAULT_SUITE_PATH, ctx="suite") + scene_ids = [ + self._CONFIG_KV_SCENE_ID, + self._CONFIG_FS_SCENE_ID, + self._CONFIG_MQ_SCENE_ID, + self._CTRL_C_KV_SCENE_ID, + self._CTRL_C_MQ_SCENE_ID, + ] + + generated = _ENTRY._rewrite_suite_for_local_dual_nodes( + suite_cfg=suite_cfg, + scene_ids=scene_ids, + primary_node_name="local-node-a", + secondary_node_name="local-node-b", + host_ip="10.1.1.119", + wheel_name="fluxon-0.2.1-cp38-abi3-manylinux_2_28_x86_64.whl", + controller_port=19080, + ) + + self.assertEqual(set(generated["scenes"].keys()), set(scene_ids)) + self.assertEqual(set(generated["scales"].keys()), {"n1_kvowner_dram_3gib"}) + self.assertEqual( + generated["profiles"]["fluxon_tcp_thread"]["runtime"]["ci"]["scene_configs"][self._CONFIG_KV_SCENE_ID], + {}, + ) + self.assertEqual( + generated["profiles"]["fluxon_tcp_thread"]["runtime"]["ci"]["scene_configs"][self._CTRL_C_MQ_SCENE_ID], + {}, + ) + self.assertEqual( + generated["scenes"][self._CONFIG_MQ_SCENE_ID]["ci"]["requirements"], + ["ci_runner", "master", "owner_0", "testbed_etcd", "testbed_greptime"], + ) + self.assertEqual( + generated["scenes"][self._CTRL_C_KV_SCENE_ID]["ci"]["requirements"], + ["ci_runner"], + ) + self.assertEqual( + generated["scenes"][self._CTRL_C_MQ_SCENE_ID]["ci"]["requirements"], + ["ci_runner", "testbed_etcd", "testbed_greptime"], + ) + def test_generated_deployconf_rewrites_to_dual_local_nodes(self) -> None: deployconf_cfg = _ENTRY._load_yaml_mapping(_ENTRY.DEFAULT_DEPLOYCONF_TEMPLATE, ctx="deployconf") generated = _ENTRY._rewrite_deployconf_for_local_dual_nodes( diff --git a/fluxon_test_stack/tests/test_test_runner_testbed_contract.py b/fluxon_test_stack/tests/test_test_runner_testbed_contract.py index 6910acd..de55b7c 100644 --- a/fluxon_test_stack/tests/test_test_runner_testbed_contract.py +++ b/fluxon_test_stack/tests/test_test_runner_testbed_contract.py @@ -65,7 +65,7 @@ def test_write_ci_scene_config_yaml_emits_structured_scene_config(self) -> None: }, "scene_config": { "doc_site_base_url": "tele-ai.github.io/Fluxon", - } + }, } }, } @@ -104,6 +104,35 @@ def test_top_attention_ci_execution_plan_is_runner_native(self) -> None: self.assertEqual(planned[0].ci_commands[0]["id"], "top_attention_bin_kvtest") self.assertIn("--case-config __RUN_DIR__/configs/ci_scene_config.yaml", planned[0].ci_commands[0]["command"]) + def test_top_attention_config_and_ctrl_c_ci_execution_plan_is_runner_native(self) -> None: + suite_cfg = yaml.safe_load((_RUNNER.RUNNER_REPO_ROOT / "fluxon_test_stack" / "ci_test_list.yaml").read_text(encoding="utf-8")) + artifact_sets = suite_cfg.get("artifact_sets") + if isinstance(artifact_sets, dict): + for artifact_set in artifact_sets.values(): + if not isinstance(artifact_set, dict): + continue + release_artifacts = artifact_set.get("release_artifacts") + if isinstance(release_artifacts, dict): + python_wheel = release_artifacts.get("python_wheel") + if isinstance(python_wheel, str) and python_wheel.strip(): + artifact_set["release_artifacts"] = {"wheel": python_wheel} + suite = _RUNNER._parse_suite_config(suite_cfg) + cases = _RUNNER._expand_cases(suite) + expected = { + "ci_top_attention_config_kv": "top_attention_config_kv", + "ci_top_attention_config_fs": "top_attention_config_fs", + "ci_top_attention_config_mq": "top_attention_config_mq", + "ci_top_attention_ctrl_c_kv": "top_attention_ctrl_c_kv", + "ci_top_attention_ctrl_c_mq": "top_attention_ctrl_c_mq", + } + + for scene_id, command_id in expected.items(): + case = next(item for item in cases if item.scene_id == scene_id and item.profile_id == "fluxon_tcp") + planned = _RUNNER._build_ci_execution_plan(case, suite) + self.assertEqual(len(planned), 1) + self.assertEqual(planned[0].ci_commands[0]["id"], command_id) + self.assertIn("--case-config __RUN_DIR__/configs/ci_scene_config.yaml", planned[0].ci_commands[0]["command"]) + def test_doc_page_ci_execution_plan_uses_online_docker_image(self) -> None: suite_cfg = yaml.safe_load((_RUNNER.RUNNER_REPO_ROOT / "fluxon_test_stack" / "ci_test_list.yaml").read_text(encoding="utf-8")) artifact_sets = suite_cfg.get("artifact_sets") @@ -234,8 +263,7 @@ def test_ci_runner_script_sources_prepare_env_when_present(self) -> None: }, "scene": { "ci": { - "subject": "doc_page", - "runtime_contract": "rust_self_managed", + "requirements": ["ci_runner"], "commands": [ { "id": "doc_page_build", @@ -493,6 +521,10 @@ def test_load_source_stack_contract_accepts_same_host_dual_local_hostworkdirs(se " ssh_user: tester", " ssh_port: 22", "service:", + " greptime:", + " port: 19295", + " node_bind:", + " node: [logic-a]", " ops_controller:", " node_bind:", " node: [logic-a]", @@ -605,6 +637,10 @@ def test_ci_base_runtime_service_target_ip_uses_loopback_for_same_host_local_nod " ssh_user: tester", " ssh_port: 22", "service:", + " greptime:", + " port: 19295", + " node_bind:", + " node: [logic-a]", " ops_controller:", " node_bind:", " node: [logic-a]", @@ -620,12 +656,7 @@ def test_ci_base_runtime_service_target_ip_uses_loopback_for_same_host_local_nod "profile": { "ci": { "runtime": { - "base_runtime": { - "greptime": { - "target": "logic-a", - "endpoint": {"scheme": "http", "host_port": 19295}, - } - } + "base_runtime": {} } } }, diff --git a/fluxon_test_stack/tests/test_test_runner_top_attention_cli.py b/fluxon_test_stack/tests/test_test_runner_top_attention_cli.py index 16a999e..95a3afe 100644 --- a/fluxon_test_stack/tests/test_test_runner_top_attention_cli.py +++ b/fluxon_test_stack/tests/test_test_runner_top_attention_cli.py @@ -45,8 +45,8 @@ def test_top_attention_list_text_prefix(self) -> None: ) self.assertEqual(completed.returncode, 0, msg=completed.stderr) requirements = {line.strip() for line in completed.stdout.splitlines() if line.strip()} - self.assertIn("ops", requirements) - self.assertIn("kv-cluster", requirements) + self.assertIn("ci_runner", requirements) + self.assertIn("master", requirements) def test_top_attention_run_requires_selector(self) -> None: completed = self.run_runner("--action", "top_attention_run") diff --git a/fluxon_test_stack/tests/test_top_attention_index_helper.py b/fluxon_test_stack/tests/test_top_attention_index_helper.py index dcec087..6fd0d72 100644 --- a/fluxon_test_stack/tests/test_top_attention_index_helper.py +++ b/fluxon_test_stack/tests/test_top_attention_index_helper.py @@ -19,7 +19,7 @@ select_top_attention_entries, top_attention_scene_id, ) -from fluxon_test_stack.top_attention_test_index.requirements_all import iter_index_entry_paths +from fluxon_test_stack.top_attention_test_index.test_test_requirements import iter_index_entry_paths class TestTopAttentionIndexHelper(unittest.TestCase): @@ -45,8 +45,8 @@ def test_collect_payload_reports_requirements(self) -> None: payload = collect_top_attention_payload(["_config_kv"]) self.assertEqual(payload["entry_count"], 1) self.assertEqual(payload["entries"][0]["name"], "_config_kv.py") - self.assertEqual(payload["entries"][0]["requirements"], ["ops"]) - self.assertEqual(payload["requirements"], ["ops"]) + self.assertEqual(payload["entries"][0]["requirements"], ["ci_runner"]) + self.assertEqual(payload["requirements"], ["ci_runner"]) def test_quick_entries_exist_and_match_declared_order(self) -> None: names = [path.name for path in iter_quick_entry_paths()] diff --git a/fluxon_test_stack/tests/test_top_attention_wrapper_case_config_contract.py b/fluxon_test_stack/tests/test_top_attention_wrapper_case_config_contract.py new file mode 100644 index 0000000..1b3d834 --- /dev/null +++ b/fluxon_test_stack/tests/test_top_attention_wrapper_case_config_contract.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import importlib.util +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def _load_module(relpath: str, module_name: str): + module_path = REPO_ROOT / relpath + module_dir = module_path.parent + sys.path.insert(0, str(module_dir)) + try: + spec = importlib.util.spec_from_file_location(module_name, module_path) + assert spec is not None and spec.loader is not None + mod = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = mod + spec.loader.exec_module(mod) + return mod + finally: + if sys.path and sys.path[0] == str(module_dir): + sys.path.pop(0) + + +class TestTopAttentionWrapperCaseConfigContract(unittest.TestCase): + def _case_cfg_path(self, *, scene_id: str) -> Path: + td = tempfile.TemporaryDirectory() + self.addCleanup(td.cleanup) + cfg_path = Path(td.name) / "ci_scene_config.yaml" + cfg_path.write_text( + yaml.safe_dump( + { + "case": { + "scene_id": scene_id, + "scale_id": "n1_kvowner_dram_3gib", + "profile_id": "fluxon_tcp_thread", + "case_id": f"{scene_id}__n1_kvowner_dram_3gib__fluxon_tcp_thread", + }, + "scene_config": {}, + "scene_runtime": { + "etcd": {"ip": "127.0.0.1", "port": 19180}, + "greptime": {"ip": "127.0.0.1", "port": 19190}, + }, + }, + sort_keys=False, + ), + encoding="utf-8", + ) + return cfg_path + + def test_config_kv_wrapper_accepts_case_config_without_forwarding_it(self) -> None: + entry = _load_module( + "fluxon_test_stack/top_attention_test_index/_config_kv.py", + "fluxon_test_stack_top_attention_config_kv_contract", + ) + cfg_path = self._case_cfg_path(scene_id="ci_top_attention_config_kv") + with ( + mock.patch.object(entry, "run_python_file", return_value=0) as run_python_file, + mock.patch.object(sys, "argv", ["_config_kv.py", "--case-config", str(cfg_path)]), + ): + self.assertEqual(entry.main(), 0) + self.assertEqual(run_python_file.call_args.kwargs["expected_scene_id"], "ci_top_attention_config_kv") + + def test_config_mq_wrapper_accepts_case_config_without_forwarding_it(self) -> None: + entry = _load_module( + "fluxon_test_stack/top_attention_test_index/_config_mq.py", + "fluxon_test_stack_top_attention_config_mq_contract", + ) + cfg_path = self._case_cfg_path(scene_id="ci_top_attention_config_mq") + with ( + mock.patch.object(entry, "run_pytest", return_value=0) as run_pytest, + mock.patch.object(sys, "argv", ["_config_mq.py", "--case-config", str(cfg_path)]), + ): + self.assertEqual(entry.main(), 0) + self.assertEqual(run_pytest.call_args.kwargs["expected_scene_id"], "ci_top_attention_config_mq") + + def test_ctrl_c_mq_wrapper_accepts_case_config_without_forwarding_it(self) -> None: + entry = _load_module( + "fluxon_test_stack/top_attention_test_index/_ctrl_c_mq.py", + "fluxon_test_stack_top_attention_ctrl_c_mq_contract", + ) + cfg_path = self._case_cfg_path(scene_id="ci_top_attention_ctrl_c_mq") + with ( + mock.patch.object(entry, "run_python_file", return_value=0) as run_python_file, + mock.patch.object(sys, "argv", ["_ctrl_c_mq.py", "--case-config", str(cfg_path)]), + ): + self.assertEqual(entry.main(), 0) + self.assertEqual(run_python_file.call_args.kwargs["expected_scene_id"], "ci_top_attention_ctrl_c_mq") + + +if __name__ == "__main__": + raise SystemExit(unittest.main()) diff --git a/fluxon_test_stack/top_attention_index_helper.py b/fluxon_test_stack/top_attention_index_helper.py index 24bc465..4ad2858 100644 --- a/fluxon_test_stack/top_attention_index_helper.py +++ b/fluxon_test_stack/top_attention_index_helper.py @@ -5,23 +5,14 @@ from pathlib import Path from typing import Any, Iterable, Sequence -try: - from fluxon_test_stack.top_attention_test_index.requirements_all import ( - TEST_REQUIREMENT_DESCRIPTIONS, - extract_test_requirements, - iter_index_entry_paths, - ) -except ModuleNotFoundError: - from top_attention_test_index.requirements_all import ( # type: ignore[no-redef] - TEST_REQUIREMENT_DESCRIPTIONS, - extract_test_requirements, - iter_index_entry_paths, - ) +import yaml REPO_ROOT = Path(__file__).resolve().parent.parent TOP_ATTENTION_INDEX_DIR = REPO_ROOT / "fluxon_test_stack" / "top_attention_test_index" +TOP_ATTENTION_SUITE_PATH = REPO_ROOT / "fluxon_test_stack" / "ci_test_list.yaml" TOP_ATTENTION_SCENE_ID_PREFIX = "ci_top_attention_" +IGNORED_INDEX_ENTRY_NAMES: frozenset[str] = frozenset({"_common.py"}) QUICK_ENTRY_NAMES: tuple[str, ...] = ( "_doc_page_build.py", "_config_kv.py", @@ -33,6 +24,33 @@ "_script_tools.py", "_cargo_fs_core.py", ) +TEST_REQUIREMENT_DESCRIPTIONS: dict[str, str] = { + "cargo": "Rust cargo toolchain is required.", + "docker": "A working Docker daemon is required.", + "fluxon-pyo3": "The compiled fluxon_pyo3 Python extension must be available.", + "fluxon-release": "The local fluxon_release runtime/artifact tree must be populated.", + "kv-cluster": "A configured KV backend runtime from the repo test config must be reachable.", + "ops": "A reachable Fluxon Ops control plane is required by the test-stack execution flow.", + "python-wheel-build": "Python wheel build dependencies must be available.", + "submodules": "Required git submodules must be initialized for build-using tests.", + "test-stack-targets": "A TEST_STACK config with reachable target hosts is required.", + "tikv": "A TiKV/PD runtime is required, either external or started by the test.", + "testbed_etcd": "The shared testbed etcd service must already be running.", + "testbed_greptime": "The shared testbed Greptime service must already be running.", + "master": "The runner must start a Fluxon master instance for the case.", + "owner_0": "The runner must start owner_0 for the case.", + "ci_runner": "The runner must start the ci_runner workload for the case.", + "owner_shared_bundle": "The runner must wait for owner shared bundle files before executing the case.", + "fluxon_kv_readiness_probe": "The runner must pass the configured Fluxon KV readiness probe before executing the case.", +} + + +def iter_index_entry_paths() -> tuple[Path, ...]: + return tuple( + path + for path in sorted(TOP_ATTENTION_INDEX_DIR.glob("*.py")) + if path.name.startswith("_") and path.name not in IGNORED_INDEX_ENTRY_NAMES + ) def display_top_attention_relpath(path: Path) -> str: @@ -46,6 +64,36 @@ def top_attention_scene_id(path: Path) -> str: return TOP_ATTENTION_SCENE_ID_PREFIX + path.stem.lstrip("_") +def _load_top_attention_suite_scenes() -> dict[str, Any]: + raw = yaml.safe_load(TOP_ATTENTION_SUITE_PATH.read_text(encoding="utf-8")) + if not isinstance(raw, dict): + raise ValueError(f"top-attention suite must be a YAML mapping: {TOP_ATTENTION_SUITE_PATH}") + scenes = raw.get("scenes") + if not isinstance(scenes, dict): + raise ValueError(f"top-attention suite scenes must be a mapping: {TOP_ATTENTION_SUITE_PATH}") + return scenes + + +def _scene_requirements_for_path(path: Path) -> list[str]: + scene_id = top_attention_scene_id(path) + scenes = _load_top_attention_suite_scenes() + scene = scenes.get(scene_id) + if not isinstance(scene, dict): + return [] + ci = scene.get("ci") + if not isinstance(ci, dict): + return [] + requirements = ci.get("requirements") + if not isinstance(requirements, list): + return [] + out: list[str] = [] + for index, raw_requirement in enumerate(requirements): + if not isinstance(raw_requirement, str) or not raw_requirement.strip(): + raise ValueError(f"scene[{scene_id}].ci.requirements[{index}] must be a non-empty string") + out.append(raw_requirement.strip()) + return sorted(set(out)) + + def match_top_attention_prefix(path: Path, raw_prefix: str) -> bool: prefix = raw_prefix.strip() if not prefix: @@ -82,7 +130,7 @@ def select_top_attention_entries(prefixes: Sequence[str]) -> list[Path]: def collect_top_attention_requirements(paths: Iterable[Path]) -> list[str]: requirements: set[str] = set() for path in paths: - requirements.update(extract_test_requirements(path)) + requirements.update(_scene_requirements_for_path(path)) return sorted(requirements) @@ -95,7 +143,7 @@ def collect_top_attention_payload(prefixes: Sequence[str] | None = None) -> dict "id": path.stem, "name": path.name, "path": display_top_attention_relpath(path), - "requirements": sorted(extract_test_requirements(path)), + "requirements": _scene_requirements_for_path(path), } ) return { @@ -145,11 +193,13 @@ def print_top_attention_payload(payload: dict[str, Any], *, requirements_only: b __all__ = [ + "IGNORED_INDEX_ENTRY_NAMES", "QUICK_ENTRY_NAMES", "TOP_ATTENTION_SCENE_ID_PREFIX", "collect_top_attention_payload", "collect_top_attention_requirements", "display_top_attention_relpath", + "iter_index_entry_paths", "iter_quick_entry_paths", "match_top_attention_prefix", "print_top_attention_payload", diff --git a/fluxon_test_stack/top_attention_test_index/README.md b/fluxon_test_stack/top_attention_test_index/README.md index 516c07e..50514a2 100644 --- a/fluxon_test_stack/top_attention_test_index/README.md +++ b/fluxon_test_stack/top_attention_test_index/README.md @@ -5,11 +5,11 @@ that now lives under `fluxon_test_stack/`. The entries stay thin: they forward to canonical tests elsewhere in the repository instead of implementing new test logic. -Each `*.py` entry here declares a sorted `TEST_REQUIREMENTS` list. The full -requirement universe lives in `requirements_all.py`, and `_test_requirements.py` -checks that per-entry declarations stay within that universe and fully cover it. -All indexed items now declare `ops` because execution is expected to go through -the shared Fluxon Ops / test-stack path. +Top-attention requirement metadata now lives only in +`fluxon_test_stack/ci_test_list.yaml` under each `ci_top_attention_*` scene's +`scene.ci.requirements`. `_test_requirements.py` checks that every indexed entry +has a matching CI scene and that the YAML-backed requirement lists stay +normalized. This directory is listing-only. Runtime orchestration, status, and log UI are owned by the `fluxon_test_stack` flow, not by this index directory. Listing and diff --git a/fluxon_test_stack/top_attention_test_index/_bin_external_client.py b/fluxon_test_stack/top_attention_test_index/_bin_external_client.py index b3f0b50..9ccd14c 100755 --- a/fluxon_test_stack/top_attention_test_index/_bin_external_client.py +++ b/fluxon_test_stack/top_attention_test_index/_bin_external_client.py @@ -7,8 +7,6 @@ from _common import REPO_ROOT, run_cargo -TEST_REQUIREMENTS = ["cargo", "etcd", "ops", "submodules"] - def main() -> int: parser = argparse.ArgumentParser( diff --git a/fluxon_test_stack/top_attention_test_index/_bin_kvtest.py b/fluxon_test_stack/top_attention_test_index/_bin_kvtest.py index faddb51..3dfe01d 100644 --- a/fluxon_test_stack/top_attention_test_index/_bin_kvtest.py +++ b/fluxon_test_stack/top_attention_test_index/_bin_kvtest.py @@ -10,7 +10,6 @@ from _common import REPO_ROOT, load_case_config_payload, run_cargo -TEST_REQUIREMENTS = ["cargo", "etcd", "ops", "submodules"] SCENE_ID = "ci_top_attention_bin_kvtest" KV_TEST_ROUND_NAMES = ("p2p_only", "rdma_transfer_only", "rdma_transfer_with_rpc") diff --git a/fluxon_test_stack/top_attention_test_index/_cargo_fs_core.py b/fluxon_test_stack/top_attention_test_index/_cargo_fs_core.py index cbca6f5..0cf9bf2 100755 --- a/fluxon_test_stack/top_attention_test_index/_cargo_fs_core.py +++ b/fluxon_test_stack/top_attention_test_index/_cargo_fs_core.py @@ -4,8 +4,6 @@ from _common import REPO_ROOT, run_cargo -TEST_REQUIREMENTS = ["cargo", "ops", "submodules"] - def main() -> int: return run_cargo([ diff --git a/fluxon_test_stack/top_attention_test_index/_cargo_kv_unit.py b/fluxon_test_stack/top_attention_test_index/_cargo_kv_unit.py index 36ae5ff..8b1875b 100755 --- a/fluxon_test_stack/top_attention_test_index/_cargo_kv_unit.py +++ b/fluxon_test_stack/top_attention_test_index/_cargo_kv_unit.py @@ -7,8 +7,6 @@ from _common import REPO_ROOT, run_cargo -TEST_REQUIREMENTS = ["cargo", "etcd", "ops", "submodules"] - def main() -> int: parser = argparse.ArgumentParser( diff --git a/fluxon_test_stack/top_attention_test_index/_cargo_util.py b/fluxon_test_stack/top_attention_test_index/_cargo_util.py index 2e707c8..73c1656 100755 --- a/fluxon_test_stack/top_attention_test_index/_cargo_util.py +++ b/fluxon_test_stack/top_attention_test_index/_cargo_util.py @@ -4,8 +4,6 @@ from _common import REPO_ROOT, run_cargo -TEST_REQUIREMENTS = ["cargo", "etcd", "ops", "submodules"] - def main() -> int: return run_cargo([ diff --git a/fluxon_test_stack/top_attention_test_index/_common.py b/fluxon_test_stack/top_attention_test_index/_common.py index 3991aa7..f26953b 100755 --- a/fluxon_test_stack/top_attention_test_index/_common.py +++ b/fluxon_test_stack/top_attention_test_index/_common.py @@ -13,37 +13,73 @@ REPO_ROOT = Path(__file__).resolve().parents[2] -TEST_REQUIREMENTS: list[str] = ["ops"] - def call(cmd: Sequence[str], *, env: dict[str, str] | None = None) -> int: print("+ " + " ".join(cmd), flush=True) return subprocess.call(list(cmd), cwd=str(REPO_ROOT), env=env) -def parse_python_passthrough(description: str) -> tuple[str, list[str]]: +def parse_python_passthrough( + description: str, + *, + expected_scene_id: str | None = None, +) -> tuple[str, list[str], dict | None]: parser = argparse.ArgumentParser(description=description) parser.add_argument( "--python", default=os.environ.get("PYTHON", sys.executable), help="Python executable used for the delegated command.", ) + parser.add_argument( + "--case-config", + help="Canonical CI case config YAML emitted by test_runner.", + ) args, passthrough = parser.parse_known_args() - return args.python, passthrough - - -def run_pytest(description: str, paths: Iterable[str]) -> int: - python, passthrough = parse_python_passthrough(description) + scene_config = None + if args.case_config is not None: + if expected_scene_id is None: + raise ValueError("--case-config is only supported for CI-bound top-attention wrappers") + scene_config = load_case_config(args.case_config, expected_scene_id=expected_scene_id) + return args.python, passthrough, scene_config + + +def run_pytest( + description: str, + paths: Iterable[str], + *, + expected_scene_id: str | None = None, +) -> int: + python, passthrough, _scene_config = parse_python_passthrough( + description, + expected_scene_id=expected_scene_id, + ) return call([python, "-m", "pytest", *paths, *passthrough]) -def run_python_file(description: str, path: str, extra_args: Iterable[str] = ()) -> int: - python, passthrough = parse_python_passthrough(description) +def run_python_file( + description: str, + path: str, + extra_args: Iterable[str] = (), + *, + expected_scene_id: str | None = None, +) -> int: + python, passthrough, _scene_config = parse_python_passthrough( + description, + expected_scene_id=expected_scene_id, + ) return call([python, "-u", str(REPO_ROOT / path), *extra_args, *passthrough]) -def run_python_files(description: str, paths: Iterable[str]) -> int: - python, passthrough = parse_python_passthrough(description) +def run_python_files( + description: str, + paths: Iterable[str], + *, + expected_scene_id: str | None = None, +) -> int: + python, passthrough, _scene_config = parse_python_passthrough( + description, + expected_scene_id=expected_scene_id, + ) for path in paths: rc = call([python, "-u", str(REPO_ROOT / path), *passthrough]) if rc != 0: diff --git a/fluxon_test_stack/top_attention_test_index/_config_fs.py b/fluxon_test_stack/top_attention_test_index/_config_fs.py index 3392324..2b6f072 100755 --- a/fluxon_test_stack/top_attention_test_index/_config_fs.py +++ b/fluxon_test_stack/top_attention_test_index/_config_fs.py @@ -4,13 +4,14 @@ from _common import run_python_file -TEST_REQUIREMENTS = ["ops"] +SCENE_ID = "ci_top_attention_config_fs" def main() -> int: return run_python_file( "Flat index entry for FS Python config/schema tests.", "fluxon_py/tests/test_fluxon_fs_config_types.py", + expected_scene_id=SCENE_ID, ) diff --git a/fluxon_test_stack/top_attention_test_index/_config_kv.py b/fluxon_test_stack/top_attention_test_index/_config_kv.py index 775284b..1ba7fab 100755 --- a/fluxon_test_stack/top_attention_test_index/_config_kv.py +++ b/fluxon_test_stack/top_attention_test_index/_config_kv.py @@ -4,13 +4,14 @@ from _common import run_python_file -TEST_REQUIREMENTS = ["ops"] +SCENE_ID = "ci_top_attention_config_kv" def main() -> int: return run_python_file( "Flat index entry for KV Python config/schema tests.", "fluxon_py/tests/test_config.py", + expected_scene_id=SCENE_ID, ) diff --git a/fluxon_test_stack/top_attention_test_index/_config_mq.py b/fluxon_test_stack/top_attention_test_index/_config_mq.py index 9930491..5294a4c 100755 --- a/fluxon_test_stack/top_attention_test_index/_config_mq.py +++ b/fluxon_test_stack/top_attention_test_index/_config_mq.py @@ -4,7 +4,7 @@ from _common import run_pytest -TEST_REQUIREMENTS = ["etcd", "fluxon-pyo3", "kv-cluster", "ops", "submodules"] +SCENE_ID = "ci_top_attention_config_mq" def main() -> int: @@ -14,6 +14,7 @@ def main() -> int: "fluxon_py/tests/test_mq/test_capacity_and_auto_clean.py", "fluxon_py/tests/test_mq/test_payload_lease_error.py", ], + expected_scene_id=SCENE_ID, ) diff --git a/fluxon_test_stack/top_attention_test_index/_ctrl_c_kv.py b/fluxon_test_stack/top_attention_test_index/_ctrl_c_kv.py index f3e5e75..1b1b0f9 100755 --- a/fluxon_test_stack/top_attention_test_index/_ctrl_c_kv.py +++ b/fluxon_test_stack/top_attention_test_index/_ctrl_c_kv.py @@ -4,7 +4,7 @@ from _common import run_python_file -TEST_REQUIREMENTS = ["ops"] +SCENE_ID = "ci_top_attention_ctrl_c_kv" def main() -> int: @@ -12,6 +12,7 @@ def main() -> int: "Flat index entry for existing KV/runtime Ctrl-C shutdown coverage.", "fluxon_py/tests/test_process_runner.py", ["TestProcessRunner.test_wait_subproc_or_ctrlc_retires_children_on_sigterm"], + expected_scene_id=SCENE_ID, ) diff --git a/fluxon_test_stack/top_attention_test_index/_ctrl_c_mq.py b/fluxon_test_stack/top_attention_test_index/_ctrl_c_mq.py index 3e56e8a..7f1ef67 100755 --- a/fluxon_test_stack/top_attention_test_index/_ctrl_c_mq.py +++ b/fluxon_test_stack/top_attention_test_index/_ctrl_c_mq.py @@ -4,13 +4,14 @@ from _common import run_python_file -TEST_REQUIREMENTS = ["etcd", "fluxon-pyo3", "fluxon-release", "greptime", "ops", "submodules"] +SCENE_ID = "ci_top_attention_ctrl_c_mq" def main() -> int: return run_python_file( "Flat index entry for existing MQ Ctrl-C integration coverage.", "fluxon_py/tests/test_mq/test_example_ctrl_c_exit.py", + expected_scene_id=SCENE_ID, ) diff --git a/fluxon_test_stack/top_attention_test_index/_deployment_codegen.py b/fluxon_test_stack/top_attention_test_index/_deployment_codegen.py index 8bac6eb..cdba174 100755 --- a/fluxon_test_stack/top_attention_test_index/_deployment_codegen.py +++ b/fluxon_test_stack/top_attention_test_index/_deployment_codegen.py @@ -4,8 +4,6 @@ from _common import run_python_files -TEST_REQUIREMENTS = ["ops"] - def main() -> int: return run_python_files( diff --git a/fluxon_test_stack/top_attention_test_index/_doc_page_build.py b/fluxon_test_stack/top_attention_test_index/_doc_page_build.py index 8047045..b336607 100644 --- a/fluxon_test_stack/top_attention_test_index/_doc_page_build.py +++ b/fluxon_test_stack/top_attention_test_index/_doc_page_build.py @@ -8,7 +8,6 @@ from _common import REPO_ROOT, call, load_case_config -TEST_REQUIREMENTS = ["ops"] SCENE_ID = "ci_top_attention_doc_page_build" diff --git a/fluxon_test_stack/top_attention_test_index/_fs_py_core.py b/fluxon_test_stack/top_attention_test_index/_fs_py_core.py index fa53e10..9f9f35b 100755 --- a/fluxon_test_stack/top_attention_test_index/_fs_py_core.py +++ b/fluxon_test_stack/top_attention_test_index/_fs_py_core.py @@ -4,8 +4,6 @@ from _common import run_pytest -TEST_REQUIREMENTS = ["etcd", "fluxon-pyo3", "kv-cluster", "ops", "submodules"] - def main() -> int: return run_pytest( diff --git a/fluxon_test_stack/top_attention_test_index/_fs_remote_mount.py b/fluxon_test_stack/top_attention_test_index/_fs_remote_mount.py index 4cae6e4..c51075c 100755 --- a/fluxon_test_stack/top_attention_test_index/_fs_remote_mount.py +++ b/fluxon_test_stack/top_attention_test_index/_fs_remote_mount.py @@ -4,8 +4,6 @@ from _common import run_pytest -TEST_REQUIREMENTS = ["etcd", "fluxon-pyo3", "kv-cluster", "ops", "submodules"] - def main() -> int: return run_pytest( diff --git a/fluxon_test_stack/top_attention_test_index/_fs_transfer_tikv.py b/fluxon_test_stack/top_attention_test_index/_fs_transfer_tikv.py index cdc685c..559311e 100755 --- a/fluxon_test_stack/top_attention_test_index/_fs_transfer_tikv.py +++ b/fluxon_test_stack/top_attention_test_index/_fs_transfer_tikv.py @@ -4,8 +4,6 @@ from _common import run_pytest -TEST_REQUIREMENTS = ["etcd", "fluxon-pyo3", "fluxon-release", "ops", "submodules", "tikv"] - def main() -> int: return run_pytest( diff --git a/fluxon_test_stack/top_attention_test_index/_kv_py_core.py b/fluxon_test_stack/top_attention_test_index/_kv_py_core.py index 924ca7a..8168709 100755 --- a/fluxon_test_stack/top_attention_test_index/_kv_py_core.py +++ b/fluxon_test_stack/top_attention_test_index/_kv_py_core.py @@ -4,8 +4,6 @@ from _common import run_pytest -TEST_REQUIREMENTS = ["etcd", "fluxon-pyo3", "kv-cluster", "ops", "submodules"] - def main() -> int: return run_pytest( diff --git a/fluxon_test_stack/top_attention_test_index/_largescale_mq.py b/fluxon_test_stack/top_attention_test_index/_largescale_mq.py index 7211bd9..6df816c 100755 --- a/fluxon_test_stack/top_attention_test_index/_largescale_mq.py +++ b/fluxon_test_stack/top_attention_test_index/_largescale_mq.py @@ -14,8 +14,6 @@ from _common import REPO_ROOT, call -TEST_REQUIREMENTS = ["fluxon-release", "ops", "submodules", "test-stack-targets"] - SCENE_ID = "bench_mq" DEFAULT_PROFILE_ID = "fluxon_tcp" diff --git a/fluxon_test_stack/top_attention_test_index/_mq_core.py b/fluxon_test_stack/top_attention_test_index/_mq_core.py index 4da1b14..6c2196e 100755 --- a/fluxon_test_stack/top_attention_test_index/_mq_core.py +++ b/fluxon_test_stack/top_attention_test_index/_mq_core.py @@ -4,8 +4,6 @@ from _common import run_pytest -TEST_REQUIREMENTS = ["etcd", "fluxon-pyo3", "kv-cluster", "ops", "submodules"] - def main() -> int: return run_pytest( diff --git a/fluxon_test_stack/top_attention_test_index/_mq_mpmc.py b/fluxon_test_stack/top_attention_test_index/_mq_mpmc.py index 0b3db0e..3b9ba2c 100755 --- a/fluxon_test_stack/top_attention_test_index/_mq_mpmc.py +++ b/fluxon_test_stack/top_attention_test_index/_mq_mpmc.py @@ -4,8 +4,6 @@ from _common import run_pytest -TEST_REQUIREMENTS = ["etcd", "kv-cluster", "ops"] - def main() -> int: return run_pytest( diff --git a/fluxon_test_stack/top_attention_test_index/_mq_mpmc_bench.py b/fluxon_test_stack/top_attention_test_index/_mq_mpmc_bench.py index 8c6501b..a2e9231 100755 --- a/fluxon_test_stack/top_attention_test_index/_mq_mpmc_bench.py +++ b/fluxon_test_stack/top_attention_test_index/_mq_mpmc_bench.py @@ -4,8 +4,6 @@ from _common import run_pytest -TEST_REQUIREMENTS = ["etcd", "kv-cluster", "ops"] - def main() -> int: return run_pytest( diff --git a/fluxon_test_stack/top_attention_test_index/_mq_mpsc.py b/fluxon_test_stack/top_attention_test_index/_mq_mpsc.py index 3ff94ef..1c2ec60 100755 --- a/fluxon_test_stack/top_attention_test_index/_mq_mpsc.py +++ b/fluxon_test_stack/top_attention_test_index/_mq_mpsc.py @@ -4,8 +4,6 @@ from _common import run_pytest -TEST_REQUIREMENTS = ["etcd", "kv-cluster", "ops"] - def main() -> int: return run_pytest( diff --git a/fluxon_test_stack/top_attention_test_index/_pack_test_rsc.py b/fluxon_test_stack/top_attention_test_index/_pack_test_rsc.py index 355f380..010b4aa 100755 --- a/fluxon_test_stack/top_attention_test_index/_pack_test_rsc.py +++ b/fluxon_test_stack/top_attention_test_index/_pack_test_rsc.py @@ -4,8 +4,6 @@ from _common import run_python_file -TEST_REQUIREMENTS = ["fluxon-release", "ops", "submodules"] - def main() -> int: return run_python_file( diff --git a/fluxon_test_stack/top_attention_test_index/_pack_whl.py b/fluxon_test_stack/top_attention_test_index/_pack_whl.py index 1106968..1eac186 100755 --- a/fluxon_test_stack/top_attention_test_index/_pack_whl.py +++ b/fluxon_test_stack/top_attention_test_index/_pack_whl.py @@ -4,8 +4,6 @@ from _common import REPO_ROOT, call, parse_python_passthrough -TEST_REQUIREMENTS = ["ops", "python-wheel-build", "submodules"] - def main() -> int: python, passthrough = parse_python_passthrough( diff --git a/fluxon_test_stack/top_attention_test_index/_py_runtime.py b/fluxon_test_stack/top_attention_test_index/_py_runtime.py index b9de962..e6067ed 100755 --- a/fluxon_test_stack/top_attention_test_index/_py_runtime.py +++ b/fluxon_test_stack/top_attention_test_index/_py_runtime.py @@ -4,8 +4,6 @@ from _common import run_python_files -TEST_REQUIREMENTS = ["ops"] - def main() -> int: return run_python_files( diff --git a/fluxon_test_stack/top_attention_test_index/_relay_mq.py b/fluxon_test_stack/top_attention_test_index/_relay_mq.py index a9d07f6..2fe519f 100755 --- a/fluxon_test_stack/top_attention_test_index/_relay_mq.py +++ b/fluxon_test_stack/top_attention_test_index/_relay_mq.py @@ -4,8 +4,6 @@ from _common import run_pytest -TEST_REQUIREMENTS = ["docker", "etcd", "fluxon-release", "ops", "submodules"] - def main() -> int: return run_pytest( diff --git a/fluxon_test_stack/top_attention_test_index/_script_tools.py b/fluxon_test_stack/top_attention_test_index/_script_tools.py index 304b7b9..4c450ab 100755 --- a/fluxon_test_stack/top_attention_test_index/_script_tools.py +++ b/fluxon_test_stack/top_attention_test_index/_script_tools.py @@ -4,8 +4,6 @@ from _common import run_python_files -TEST_REQUIREMENTS = ["ops"] - def main() -> int: return run_python_files( diff --git a/fluxon_test_stack/top_attention_test_index/_test_requirements.py b/fluxon_test_stack/top_attention_test_index/_test_requirements.py index ddf87a6..3868bd7 100644 --- a/fluxon_test_stack/top_attention_test_index/_test_requirements.py +++ b/fluxon_test_stack/top_attention_test_index/_test_requirements.py @@ -4,12 +4,10 @@ from _common import run_python_file -TEST_REQUIREMENTS = ["ops"] - def main() -> int: return run_python_file( - "Flat index entry for TEST_REQUIREMENTS metadata convergence checks.", + "Flat index entry for top-attention requirements metadata convergence checks.", "fluxon_test_stack/top_attention_test_index/test_test_requirements.py", ) diff --git a/fluxon_test_stack/top_attention_test_index/_test_stack_contract.py b/fluxon_test_stack/top_attention_test_index/_test_stack_contract.py index ddd6e17..fc8757c 100755 --- a/fluxon_test_stack/top_attention_test_index/_test_stack_contract.py +++ b/fluxon_test_stack/top_attention_test_index/_test_stack_contract.py @@ -4,8 +4,6 @@ from _common import run_python_file -TEST_REQUIREMENTS = ["ops"] - def main() -> int: return run_python_file( diff --git a/fluxon_test_stack/top_attention_test_index/requirements_all.py b/fluxon_test_stack/top_attention_test_index/requirements_all.py deleted file mode 100644 index edf8d87..0000000 --- a/fluxon_test_stack/top_attention_test_index/requirements_all.py +++ /dev/null @@ -1,93 +0,0 @@ -from __future__ import annotations - -import ast -from pathlib import Path - - -TEST_REQUIREMENTS: list[str] = ["ops"] -TEST_DIR = Path(__file__).resolve().parent -IGNORED_INDEX_ENTRY_NAMES = frozenset({"_common.py"}) - - -ALL_TEST_REQUIREMENTS: tuple[str, ...] = ( - "cargo", - "docker", - "etcd", - "fluxon-pyo3", - "fluxon-release", - "greptime", - "kv-cluster", - "ops", - "python-wheel-build", - "submodules", - "test-stack-targets", - "tikv", -) - -ALL_TEST_REQUIREMENTS_SET = frozenset(ALL_TEST_REQUIREMENTS) - -TEST_REQUIREMENT_DESCRIPTIONS: dict[str, str] = { - "cargo": "Rust cargo toolchain is required.", - "docker": "A working Docker daemon is required.", - "etcd": "An etcd runtime is required, either external or started by the test.", - "fluxon-pyo3": "The compiled fluxon_pyo3 Python extension must be available.", - "fluxon-release": "The local fluxon_release runtime/artifact tree must be populated.", - "greptime": "A GreptimeDB runtime is required, either external or started by the test.", - "kv-cluster": "A configured KV backend runtime from the repo test config must be reachable.", - "ops": "A reachable Fluxon Ops control plane is required by the test-stack execution flow.", - "python-wheel-build": "Python wheel build dependencies must be available.", - "submodules": "Required git submodules must be initialized for build-using tests.", - "test-stack-targets": "A TEST_STACK config with reachable target hosts is required.", - "tikv": "A TiKV/PD runtime is required, either external or started by the test.", -} - - -def iter_test_python_paths() -> tuple[Path, ...]: - return tuple(sorted(TEST_DIR.glob("*.py"))) - - -def iter_index_entry_paths() -> tuple[Path, ...]: - return tuple( - path - for path in iter_test_python_paths() - if path.name.startswith("_") and path.name not in IGNORED_INDEX_ENTRY_NAMES - ) - - -def extract_test_requirements(path: Path) -> list[str]: - tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - found = None - for node in tree.body: - target_name = None - value_node = None - if isinstance(node, ast.Assign): - for target in node.targets: - if isinstance(target, ast.Name) and target.id == "TEST_REQUIREMENTS": - target_name = target.id - value_node = node.value - break - elif isinstance(node, ast.AnnAssign): - if isinstance(node.target, ast.Name) and node.target.id == "TEST_REQUIREMENTS": - target_name = node.target.id - value_node = node.value - - if target_name != "TEST_REQUIREMENTS": - continue - if found is not None: - raise AssertionError(f"{path.name} defines TEST_REQUIREMENTS more than once") - if value_node is None: - raise AssertionError(f"{path.name} TEST_REQUIREMENTS must assign a list literal") - try: - value = ast.literal_eval(value_node) - except (ValueError, SyntaxError) as exc: - raise AssertionError( - f"{path.name} TEST_REQUIREMENTS must be a static list literal" - ) from exc - if not isinstance(value, list): - raise AssertionError(f"{path.name} TEST_REQUIREMENTS must be a list literal") - if not all(isinstance(item, str) for item in value): - raise AssertionError(f"{path.name} TEST_REQUIREMENTS must contain only strings") - found = value - if found is None: - raise AssertionError(f"{path.name} is missing TEST_REQUIREMENTS") - return found diff --git a/fluxon_test_stack/top_attention_test_index/test_test_requirements.py b/fluxon_test_stack/top_attention_test_index/test_test_requirements.py index 2c9a4bb..f3e0aaf 100644 --- a/fluxon_test_stack/top_attention_test_index/test_test_requirements.py +++ b/fluxon_test_stack/top_attention_test_index/test_test_requirements.py @@ -3,43 +3,89 @@ import unittest from pathlib import Path -from requirements_all import ( - ALL_TEST_REQUIREMENTS, - ALL_TEST_REQUIREMENTS_SET, - extract_test_requirements, - iter_test_python_paths, -) +import yaml -TEST_REQUIREMENTS: list[str] = ["ops"] +REPO_ROOT = Path(__file__).resolve().parents[2] +INDEX_DIR = Path(__file__).resolve().parent +SUITE_PATH = REPO_ROOT / "fluxon_test_stack" / "ci_test_list.yaml" +TOP_ATTENTION_SCENE_ID_PREFIX = "ci_top_attention_" +IGNORED_INDEX_ENTRY_NAMES = frozenset({"_common.py"}) -class TestTestRequirements(unittest.TestCase): - def test_every_test_py_declares_requirements(self) -> None: - for path in iter_test_python_paths(): - with self.subTest(path=path.name): - _ = extract_test_requirements(path) +def iter_index_entry_paths() -> tuple[Path, ...]: + return tuple( + path + for path in sorted(INDEX_DIR.glob("*.py")) + if path.name.startswith("_") and path.name not in IGNORED_INDEX_ENTRY_NAMES + ) + + +def top_attention_scene_id(path: Path) -> str: + return TOP_ATTENTION_SCENE_ID_PREFIX + path.stem.lstrip("_") + + +def _load_suite_scenes() -> dict[str, object]: + raw = yaml.safe_load(SUITE_PATH.read_text(encoding="utf-8")) + if not isinstance(raw, dict): + raise AssertionError(f"suite must be a mapping: {SUITE_PATH}") + scenes = raw.get("scenes") + if not isinstance(scenes, dict): + raise AssertionError(f"suite scenes must be a mapping: {SUITE_PATH}") + return scenes - def test_declared_requirement_lists_are_sorted_unique_and_known(self) -> None: - for path in iter_test_python_paths(): - requirements = extract_test_requirements(path) - with self.subTest(path=path.name): - self.assertEqual(requirements, sorted(set(requirements))) - self.assertTrue(set(requirements).issubset(ALL_TEST_REQUIREMENTS_SET)) - def test_requirement_universe_is_sorted_unique(self) -> None: - self.assertEqual(list(ALL_TEST_REQUIREMENTS), sorted(set(ALL_TEST_REQUIREMENTS))) +def _ci_scene_ids() -> set[str]: + return { + scene_id + for scene_id, raw_scene in _load_suite_scenes().items() + if isinstance(scene_id, str) + and isinstance(raw_scene, dict) + and isinstance(raw_scene.get("ci"), dict) + } - def test_requirement_universe_matches_declared_union(self) -> None: - declared_union: set[str] = set() - for path in iter_test_python_paths(): - declared_union.update(extract_test_requirements(path)) - self.assertEqual(declared_union, set(ALL_TEST_REQUIREMENTS)) + +def _scene_requirements(scene_id: str) -> list[str]: + scenes = _load_suite_scenes() + scene = scenes.get(scene_id) + if not isinstance(scene, dict): + raise AssertionError(f"missing top-attention scene in suite: {scene_id}") + ci = scene.get("ci") + if not isinstance(ci, dict): + raise AssertionError(f"top-attention scene must be CI-backed: {scene_id}") + requirements = ci.get("requirements") + if not isinstance(requirements, list): + raise AssertionError(f"scene[{scene_id}].ci.requirements must be a list") + out: list[str] = [] + for index, raw_item in enumerate(requirements): + if not isinstance(raw_item, str) or not raw_item.strip(): + raise AssertionError(f"scene[{scene_id}].ci.requirements[{index}] must be a non-empty string") + out.append(raw_item.strip()) + return out + + +class TestTopAttentionYamlRequirements(unittest.TestCase): + def test_every_index_entry_has_suite_scene(self) -> None: + for path in iter_index_entry_paths(): + scene_id = top_attention_scene_id(path) + if scene_id not in _ci_scene_ids(): + continue + with self.subTest(path=path.name, scene_id=scene_id): + _ = _scene_requirements(scene_id) + + def test_scene_requirements_are_sorted_and_unique(self) -> None: + for path in iter_index_entry_paths(): + scene_id = top_attention_scene_id(path) + if scene_id not in _ci_scene_ids(): + continue + requirements = _scene_requirements(scene_id) + with self.subTest(path=path.name, scene_id=scene_id): + self.assertEqual(requirements, sorted(set(requirements))) def test_removed_direct_entrypoints_stay_removed(self) -> None: - self.assertFalse((Path(__file__).resolve().parent / "_all_quick.py").exists()) - self.assertFalse((Path(__file__).resolve().parent / "run_match_prefix.py").exists()) + self.assertFalse((INDEX_DIR / "_all_quick.py").exists()) + self.assertFalse((INDEX_DIR / "run_match_prefix.py").exists()) if __name__ == "__main__": - unittest.main() + raise SystemExit(unittest.main()) diff --git a/skills/browser-helm/SKILL.md b/skills/browser-helm/SKILL.md new file mode 100644 index 0000000..dbe1afd --- /dev/null +++ b/skills/browser-helm/SKILL.md @@ -0,0 +1,232 @@ +--- +name: browser-helm +description: Helm-only browser runtime workflow for operating Browser Helm managed tabs via `browser-helm`, with namespaced `browser` / `tab` / `page` / `picker` / `events` commands and namespaced `.tmp/browser-helm/` output conventions. +allowed-tools: Bash(*) +--- + +# 用 `browser-helm` 操作 Browser Helm 受控标签页 + +当用户想通过 **Helm-only runtime** 操作浏览器,而不是使用通用 `agent-browser` 时,使用这个 skill。 + +适用场景: + +- 需要列出已连接浏览器 / managed tab +- 需要创建 managed tab 并 attach debugger +- 需要执行 `page navigate` / `page click` / `page eval` / `page wait` / `page type` / `page press` / `page summary` / `page snapshot` / `page screenshot` +- 需要通过 picker 获取/清空最近一次选中元素的 metadata(无需用户粘贴 JSON) +- 需要遵守 `browser-helm` 当前的输出与落盘约定 + +不适用场景: + +- 用户明确要用通用 `agent-browser` / noVNC 工作流 +- 用户只是要解释代码,不需要运行 `browser-helm` + +## 默认工作流(新主路径) + +默认 Base URL:`http://127.0.0.1:5181`(不需要设置环境变量)。 + +如需覆盖(可选):在命令前追加 `--base-url http://127.0.0.1:5181`。 + +如本机未全局安装 `browser-helm`,也可以用 `node browser-helm/dist/cli.js` 替代下方命令。 + +## 多人/多 AI 会话(互信)约定(重要) + +当前产品定位下,daemon / Web UI / WS **默认不做鉴权**,更偏向“同一局域网多人互信”的协作模型。 + +但为了避免 **同一台浏览器 + 多个 AI 对话** 时出现“串台/误操作”,推荐强制使用 `session` 做操作隔离: + +- 每个 AI 对话固定用一个 `--session `(或设置环境变量 `BROWSER_HELM_SESSION=`) +- `session` 会隔离: + - CLI context 落盘:`.tmp/browser-helm/context.json`(default)或 `.tmp/browser-helm/sessions//context.json` + - CLI 输出落盘:`.tmp/browser-helm//...`(default)或 `.tmp/browser-helm/sessions///...` + - `tab create` 会自动加前缀:`[session:] ...`(用于人类/AI 识别归属) +- `tab list --mine` 只在非 default session 下可用(通过 note 前缀过滤“我这条会话创建的 tab”) + +注意:`session` 只是“操作习惯/隔离约定”,**不是安全边界**。知道 `managed-tab-id` 仍然能跨 session 操作;不要把端口暴露到不可信网络。 + +### 前置(必须):安装插件并配对 + +`browser-helm` 的所有浏览器动作都依赖 **Chrome 插件已连接 daemon(WebSocket)**: + +- 创建 managed tab 时建议提供 `--note `,用于描述这个 tab 的意图/用途。 + - 若省略 `--note` 且提供 URL,CLI 会自动生成:`打开页面:` + +- 若 `browser-helm browser list` 一直为空,优先判断是「插件未安装/未 Connect」而不是 CLI 出错。 + +一次性配对步骤: + +1) 启动 daemon + +```bash +browser-helm daemon ensure +``` + +(可选)如需重启: + +```bash +browser-helm daemon restart +``` + +2) 用 Chrome 打开 Web UI(用“Chrome 能访问到的地址”打开) + +- Web UI:`http://127.0.0.1:5181` +- 页面上会显示 `Pairing Code`(推荐)以及 `WS URL`/`Pairing Token`(Advanced) + +3) 安装扩展(Unpacked) + +- 在 Web UI 点击“下载插件 zip”,解压 +- 打开 `chrome://extensions`,开启开发者模式 +- 点击“加载已解压的扩展程序”,选择解压后的目录 + +4) 插件配对(Connect) + +- 打开扩展弹窗 +- 粘贴 Web UI 中的 `Pairing Code`,点击 `Connect` +- (可选)点一次 `Status` 确认连接 OK +- Advanced:也可手填 `WS URL` + `Pairing Token` + +5) CLI 验证插件已连接 + +```bash +browser-helm browser list +``` + +### 默认动作流 + +1. 确保 `Browser Helm daemon` 已启动(AI 可通过 CLI 直接启动/拉起) + +```bash +browser-helm daemon ensure +``` + +注:`daemon ensure` 会启动内置的预编译 daemon(当前提供 `linux-x64`),不要求用户安装 `cargo`。 + +2. 确认扩展已连接,并列出浏览器 + +```bash +browser-helm browser list +``` + +(推荐)3. Pin 默认 browser/tab(减少长对话遗忘成本) + +```bash +browser-helm context use-browser +browser-helm context use-tab +browser-helm context show +``` + +4. 列 tab;如无 tab,则创建新 tab + +```bash +browser-helm browser list +browser-helm tab list +browser-helm tab create https://example.com --note "说明这个 tab 的用途" +``` + +5. (可选)显式 `tab attach` debugger + +`tab create` / `page navigate` 已会自动 ensure debugger attach(用于更早捕获 network/console)。如果你准备在浏览器里手动刷新/导航,也建议先 `tab attach`。 + +```bash +browser-helm tab attach +``` + +6. 页面分析优先走返回值主路 + +```bash +browser-helm page summary +browser-helm page snapshot +``` + +7. 只有在需要留档时才显式保存 `page summary` / `page snapshot` + +```bash +browser-helm --save page summary +browser-helm --save page snapshot +``` + +8. `page screenshot` 默认会落盘;`page click` 会走受控页遮罩下的程序化点击 + +```bash +browser-helm page click '#selector' +browser-helm page click '#selector' --wait-text 'Finished working' --timeout-ms 15000 +browser-helm page eval '1+1' +browser-helm page wait --until-text 'Finished working' --timeout-ms 15000 +browser-helm page type 'div[aria-label="Composer"]' 'hello' +browser-helm page press 'Enter' +browser-helm page screenshot +``` + +9. 推荐先 `page snapshot` 生成 `@iN` refs,再用 ref 操作(类似 agent-browser 的 `@eN`) + +```bash +browser-helm page snapshot +browser-helm page click @i1 +browser-helm page type @i2 'hello' +``` + +9. 如用户在 SidePanel 做了元素选择(Start Picking),AI 可直接从 daemon 拉取最近一次选择结果 + +```bash +browser-helm picker last +browser-helm picker clear +``` + +### 交互录制(用户手动复现) + +当你需要「AI 先打开受控 tab,然后用户自己操作复现问题,再让 AI 回看」时,可以开启交互录制: + +```bash +# 记录起始时间(ms) +t0=$(date +%s%3N) + +# 开始录制(会注入监听脚本,并临时隐藏遮罩,允许用户点击/输入) +browser-helm recorder start + +# ...用户在该 tab 上手动复现... + +# 拉取复现阶段的交互/console/network 事件(按 since 过滤) +browser-helm events interaction --since $t0 --limit 2000 +browser-helm events console --since $t0 --limit 2000 +browser-helm events network --since $t0 --limit 2000 + +# 停止录制(恢复遮罩) +browser-helm recorder stop +``` + +注意:交互录制会包含 input 的原始 value(不脱敏)。仅建议在互信/本地环境使用。 + +## 输出与落盘约定 + +- `page summary`:默认只打印;传 `output-path` 或 `--save` 时,写入 `.tmp/browser-helm/summaries/` +- `page snapshot`:默认只打印;传 `output-path` 或 `--save` 时,写入 `.tmp/browser-helm/snapshots/` +- `page screenshot`:默认写入 `.tmp/browser-helm/screenshots/` +- 若使用 `--session ` / `BROWSER_HELM_SESSION=`:上述目录会自动切换到 `.tmp/browser-helm/sessions//...` +- 如用户显式提供路径,优先使用用户路径 + +## 命令参考 + +详细命令与示例见:[`browser-helm/skills/browser-helm/references/commands.md`] + +优先顺序建议: + +1. `browser list` +2. `tab list` +3. `tab create`(推荐写 `--note`;若省略且提供 URL,则自动生成 note) + - 或:`tab adopt-active`(接管当前活动 tab) +4. `tab attach` +5. `page navigate` +6. `page summary` / `page snapshot` +7. `page click` / `page screenshot` + + +## 目录约定 + +- 项目内 skill 源目录:[`browser-helm/skills/browser-helm/`] +- 仓库根入口:[`skills/browser-helm/`] + + +## 命令约定 + +- 仅支持 namespaced 命令面:`browser list`、`tab create`、`page navigate`、`picker last` 等。 +- 默认文档路径改为 namespaced 形式:`browser list`、`tab create`、`page navigate`、`events console`、`picker last`。 diff --git a/skills/browser-helm/agents/openai.yaml b/skills/browser-helm/agents/openai.yaml new file mode 100644 index 0000000..686f428 --- /dev/null +++ b/skills/browser-helm/agents/openai.yaml @@ -0,0 +1,6 @@ +interface: + display_name: Browser Helm + short_description: 通过 Browser Helm 操作 Browser Helm 受控标签页流程 + default_prompt: Use $browser-helm to inspect and operate Browser Helm managed tabs. +policy: + allow_implicit_invocation: false diff --git a/skills/browser-helm/references/commands.md b/skills/browser-helm/references/commands.md new file mode 100644 index 0000000..d22d465 --- /dev/null +++ b/skills/browser-helm/references/commands.md @@ -0,0 +1,131 @@ +# `browser-helm` 命令参考 + +## 前置(必须):插件安装与配对 + +CLI 能否操作浏览器,取决于 **Chrome 插件是否已连接 daemon(WebSocket)**。 + +最小闭环步骤: + +```bash +# 1) 启动/确保 daemon +browser-helm daemon ensure +browser-helm daemon status +browser-helm daemon restart + +# 2) 在 Chrome 打开 Web UI(用 Chrome 能访问到的地址打开) +# http://127.0.0.1:5181 +# 从页面复制 Pairing Code(推荐;含多网卡候选地址)或 WS URL + Pairing Token(Advanced) +# +# 3) 安装扩展(Unpacked) +# - Web UI 下载插件 zip -> 解压 +# - chrome://extensions 开启开发者模式 -> 加载已解压扩展 +# +# 4) 插件弹窗填 Pairing Code -> Connect + +# 5) 验证浏览器已连接 +browser-helm browser list +``` + +## 基础命令(新主路径) + +```bash +browser-helm daemon status +browser-helm daemon ensure +browser-helm daemon stop +browser-helm daemon restart +browser-helm status +browser-helm browser list +browser-helm tab list [browser-id] [--mine] +browser-helm recorder start [browser-id] [managed-tab-id] +browser-helm recorder stop [browser-id] [managed-tab-id] +``` + +## 受控 tab 生命周期 + +```bash +browser-helm tab create [browser-id] [url] [--note ] +browser-helm tab adopt-active [browser-id] [--note ] +browser-helm tab attach [browser-id] [managed-tab-id] +browser-helm page navigate [browser-id] [managed-tab-id] +``` + +## 交互与分析 + +```bash +browser-helm page click [browser-id] [managed-tab-id] [--wait-(selector|text|js) ] [--timeout-ms ] [--interval-ms ] +browser-helm page eval [browser-id] [managed-tab-id] +browser-helm page wait [browser-id] [managed-tab-id] --until-(selector|text|js) [--timeout-ms ] [--interval-ms ] +browser-helm page type [browser-id] [managed-tab-id] +browser-helm page press [browser-id] [managed-tab-id] +browser-helm page summary [browser-id] [managed-tab-id] [output-path] +browser-helm page snapshot [browser-id] [managed-tab-id] [output-path] +browser-helm page screenshot [browser-id] [managed-tab-id] [output-path] +browser-helm events console [browser-id] [managed-tab-id] [--limit ] [--since ] +browser-helm events network [browser-id] [managed-tab-id] [--limit ] [--since ] +browser-helm events interaction [browser-id] [managed-tab-id] [--limit ] [--since ] +browser-helm picker last [browser-id] [managed-tab-id] +browser-helm picker clear [browser-id] [managed-tab-id] +``` + +说明: + +- `page snapshot` 会生成可复用的 interactive refs:`@i1/@i2/...`(按 interactives 列表顺序)。 +- `page click/@iN`、`page type/@iN` 会把 ref 解析为 snapshot 中记录的 selector(落盘于 `.tmp/browser-helm/refs/.json`,按 `--session` 隔离)。 + +## Context(session-like,新主路径) + +长对话/长任务里,为了避免反复提供 `browser-id` / `managed-tab-id`,可以把默认对象写入本地 context: + +```bash +browser-helm context use-browser +browser-helm context use-tab +browser-helm context show +browser-helm context clear +``` + +## 多 AI 对话隔离(推荐) + +为了避免“同一浏览器 + 多个 AI 对话”串台,建议为每条对话固定一个 `session`: + +```bash +browser-helm --session chat-a browser list +browser-helm --session chat-a tab list --mine +browser-helm --session chat-a tab create https://example.com --note "这条对话的用途说明" +``` + +说明: + +- `tab create` 会自动加前缀:`[session:chat-a] ...` +- `tab list --mine` 需要非 default session(否则会报错) + +## 输出约定 + +- `page summary` + - 默认只打印 + - `--save` 时默认落到 [`.tmp/browser-helm/summaries/`] +- `page snapshot` + - 默认只打印 + - `--save` 时默认落到 [`.tmp/browser-helm/snapshots/`] +- `page screenshot` + - 默认落到 [`.tmp/browser-helm/screenshots/`] +- 若使用 `--session ` / `BROWSER_HELM_SESSION=`:上述目录会自动切换到 [`.tmp/browser-helm/sessions//...`] + +## 推荐示例 + +```bash +browser-helm browser list +browser-helm tab create https://example.com --note "说明这个 tab 的用途" +browser-helm tab attach +browser-helm page snapshot +browser-helm --save page summary +browser-helm page screenshot +``` + +说明: + +- `tab create` 若省略 `--note` 且提供 URL,会自动生成:`打开页面:` + +## 命令约定 + +- 仅支持 namespaced 命令面:`browser list`、`tab create`、`tab attach`、`page navigate`、`picker last` 等。 +- 文档与 skill 后续默认都以 namespaced 命令作为主路径。 diff --git a/skills/canvas-dag_organizer-v1/SKILL.md b/skills/canvas-dag_organizer-v1/SKILL.md new file mode 100644 index 0000000..db3dc0d --- /dev/null +++ b/skills/canvas-dag_organizer-v1/SKILL.md @@ -0,0 +1,10 @@ +--- +name: "canvas-dag_organizer-v1" +description: "Canvas DAG Organizer v1" +metadata: + short-description: "Canvas DAG Organizer v1" +--- + +# Canvas DAG Organizer v1 + +你是「Canvas DAG 可读性优化专家」(canvas_dag_organizer)。\n你的目标:基于当前 canvas 内容与 DAG(causal/timeline edges)结构,决定如何拆分/分组/调整空间布局,以最大化可读性。\n\n硬约束(必须遵守):\n- 禁止要求用户手工编辑 `.canvas` / `.canvas.ext` JSON。\n- 你不能执行任何命令;你只能输出一个严格 JSON 对象(不要 markdown、不要 code fence、不要额外文本)。\n- 你输出的修改必须是“可复现/确定性”的(同一输入得到同一输出)。\n\n你会收到:\n- path + expectedCanvasSha256(并发保护)\n- scopeNodes / scopeEdges(允许你改动的子图范围)\n- 每个节点的 effective rect(考虑 ext.dx/dy/scale)\n\n你的输出 JSON schema(version=1):\n{\n "version": 1,\n "kind": "canvas_dag_organize_apply_v1",\n "path": "",\n "expectedCanvasSha256": "",\n "summary": "一句话总结你做了什么(用于 UI 提示)",\n "ops": [\n // CanvasOpsRequestV1.ops: op=upsert_node|delete_node|upsert_edge|delete_edge\n ]\n}\n\n重要规则:\n- 只允许改动 scope 内的 existing session nodes(移动/尺寸/文本等)与 existing edges。\n- 允许创建 group 节点用于分区(id 必须以 "group-" 开头;type="group")。\n- 禁止删除任何 session 节点(dever_kind=session)。\n- 如果你删除 node,必须同时删除所有引用它的 edges(否则服务端会拒绝 apply)。\n- 优先做:分组 + 分层/泳道 + 对齐 + 留白;不要盲目网格化。 diff --git a/skills/canvas-dag_organizer-v1/agents/openai.yaml b/skills/canvas-dag_organizer-v1/agents/openai.yaml new file mode 100644 index 0000000..f7ffc0e --- /dev/null +++ b/skills/canvas-dag_organizer-v1/agents/openai.yaml @@ -0,0 +1,6 @@ +interface: + display_name: "Canvas DAG Organizer v1" + short_description: "Canvas DAG Organizer v1" + default_prompt: "Use $canvas-dag_organizer-v1." +policy: + allow_implicit_invocation: false diff --git a/skills/canvas-ops-v1/SKILL.md b/skills/canvas-ops-v1/SKILL.md new file mode 100644 index 0000000..ffa8017 --- /dev/null +++ b/skills/canvas-ops-v1/SKILL.md @@ -0,0 +1,10 @@ +--- +name: "canvas-ops-v1" +description: "Canvas Ops v1" +metadata: + short-description: "Canvas Ops v1" +--- + +# Canvas Ops v1 + +你是「Canvas 文件操作助手」(canvas_ops)。\n你的目标:对 `*.canvas` / `*.canvas.ext` 的任何修改,都必须通过项目内的脚本执行;禁止手工编辑 JSON。\n\n唯一允许的执行入口:\n- `.dever/tools/canvas_ops/canvas_ops.sh`\n- 配置:`.dever/tools/canvas_ops/config.json`\n\n硬约束:\n- 你只能生成 `apply` 需要的 request JSON(version=1),并给出一条可执行命令来调用脚本。\n- 禁止直接输出/粘贴完整 `.canvas` 内容作为“修改后的文件”。\n- 如果需要删除 node:必须同时显式删除所有依赖该 node 的 edges(否则脚本会拒绝执行)。\n\n你的输出格式(两段,且仅两段):\n(1) request JSON(纯 JSON,不要 markdown,不要 code fence)\n(2) 一段 bash 命令(用 heredoc 把 JSON 送进脚本;命令内必须显式传 `-w` 与 `-c`)\n\n命令模板(把 替换为项目根;一般是 `.`):\n.dever/tools/canvas_ops/canvas_ops.sh apply -w -c .dever/tools/canvas_ops/config.json --request-stdin <<'JSON'\n{...}\nJSON\n\n建议(可选):命令后再跑一次 validate,确认写盘结果可读且 ext sha 一致。 diff --git a/skills/canvas-ops-v1/agents/openai.yaml b/skills/canvas-ops-v1/agents/openai.yaml new file mode 100644 index 0000000..5566cff --- /dev/null +++ b/skills/canvas-ops-v1/agents/openai.yaml @@ -0,0 +1,6 @@ +interface: + display_name: Canvas Ops v1 + short_description: Canvas Ops v1 + default_prompt: Use $canvas-ops-v1. +policy: + allow_implicit_invocation: false diff --git a/skills/canvas-tidy_selection-v1/SKILL.md b/skills/canvas-tidy_selection-v1/SKILL.md new file mode 100644 index 0000000..0dbfdf1 --- /dev/null +++ b/skills/canvas-tidy_selection-v1/SKILL.md @@ -0,0 +1,10 @@ +--- +name: "canvas-tidy_selection-v1" +description: "Canvas Tidy Selection v1" +metadata: + short-description: "Canvas Tidy Selection v1" +--- + +# Canvas Tidy Selection v1 + +你是「Canvas 会话块整理专家」(canvas_tidy_selection)。\n你的目标:为“画布上选中的会话块”提供一键自动整理(确定性布局、可复现)。\n\n硬约束:\n- 禁止建议用户手工编辑 `.canvas` / `.canvas.ext` JSON。\n- 不要输出“修改后的完整 canvas 文件内容”。\n- 你只能输出(两段,且仅两段):\n (1) request JSON(纯 JSON,不要 markdown,不要 code fence)\n (2) 一条 curl 命令(向 manager 的 tidy_selection API 发请求)。\n\n请求/响应(V1)约定:\n- Endpoint: POST /api/projects/:projectId/canvas/tidy_selection\n- request JSON schema (version=1):\n - version: 1\n - path: string (project root 下的相对路径,必须以 .canvas 结尾)\n - expectedCanvasSha256: string (并发保护;必须来自最新 load 响应的 canvas_sha256)\n - selectedSessionIds: string[] (选中的会话块 node id 列表;会去重并保持稳定顺序)\n - layout: { kind: "grid_sqrt_v1"; gapX: number; gapY: number }\n - anchor: { kind: "keep_bounds_topleft_v1" }\n - resetConnectedEdgeRoutes: boolean (true 表示清空相关连线 ext 路由,回到默认路由)\n\ncurl 模板(把 替换为实际 id):\ncurl -sS -X POST 'http://localhost:8788/api/projects//canvas/tidy_selection' \\n -H 'Content-Type: application/json' \\n -d ''\n\n输出策略:\n- 不要向用户提问;基于已给信息直接产出最强可执行请求。\n- 若关键信息缺失(例如 projectId/path/sha/selected ids),在 request JSON 中用空值占位,并在 curl 命令中保留 <...> 占位符。 diff --git a/skills/canvas-tidy_selection-v1/agents/openai.yaml b/skills/canvas-tidy_selection-v1/agents/openai.yaml new file mode 100644 index 0000000..120f1ac --- /dev/null +++ b/skills/canvas-tidy_selection-v1/agents/openai.yaml @@ -0,0 +1,6 @@ +interface: + display_name: "Canvas Tidy Selection v1" + short_description: "Canvas Tidy Selection v1" + default_prompt: "Use $canvas-tidy_selection-v1." +policy: + allow_implicit_invocation: false diff --git a/skills/find-skills/SKILL.md b/skills/find-skills/SKILL.md new file mode 100644 index 0000000..c797184 --- /dev/null +++ b/skills/find-skills/SKILL.md @@ -0,0 +1,133 @@ +--- +name: find-skills +description: Helps users discover and install agent skills when they ask questions like "how do I do X", "find a skill for X", "is there a skill that can...", or express interest in extending capabilities. This skill should be used when the user is looking for functionality that might exist as an installable skill. +--- + +# Find Skills + +This skill helps you discover and install skills from the open agent skills ecosystem. + +## When to Use This Skill + +Use this skill when the user: + +- Asks "how do I do X" where X might be a common task with an existing skill +- Says "find a skill for X" or "is there a skill for X" +- Asks "can you do X" where X is a specialized capability +- Expresses interest in extending agent capabilities +- Wants to search for tools, templates, or workflows +- Mentions they wish they had help with a specific domain (design, testing, deployment, etc.) + +## What is the Skills CLI? + +The Skills CLI (`npx skills`) is the package manager for the open agent skills ecosystem. Skills are modular packages that extend agent capabilities with specialized knowledge, workflows, and tools. + +**Key commands:** + +- `npx skills find [query]` - Search for skills interactively or by keyword +- `npx skills add ` - Install a skill from GitHub or other sources +- `npx skills check` - Check for skill updates +- `npx skills update` - Update all installed skills + +**Browse skills at:** https://skills.sh/ + +## How to Help Users Find Skills + +### Step 1: Understand What They Need + +When a user asks for help with something, identify: + +1. The domain (e.g., React, testing, design, deployment) +2. The specific task (e.g., writing tests, creating animations, reviewing PRs) +3. Whether this is a common enough task that a skill likely exists + +### Step 2: Search for Skills + +Run the find command with a relevant query: + +```bash +npx skills find [query] +``` + +For example: + +- User asks "how do I make my React app faster?" → `npx skills find react performance` +- User asks "can you help me with PR reviews?" → `npx skills find pr review` +- User asks "I need to create a changelog" → `npx skills find changelog` + +The command will return results like: + +``` +Install with npx skills add + +vercel-labs/agent-skills@vercel-react-best-practices +└ https://skills.sh/vercel-labs/agent-skills/vercel-react-best-practices +``` + +### Step 3: Present Options to the User + +When you find relevant skills, present them to the user with: + +1. The skill name and what it does +2. The install command they can run +3. A link to learn more at skills.sh + +Example response: + +``` +I found a skill that might help! The "vercel-react-best-practices" skill provides +React and Next.js performance optimization guidelines from Vercel Engineering. + +To install it: +npx skills add vercel-labs/agent-skills@vercel-react-best-practices + +Learn more: https://skills.sh/vercel-labs/agent-skills/vercel-react-best-practices +``` + +### Step 4: Offer to Install + +If the user wants to proceed, you can install the skill for them: + +```bash +npx skills add -g -y +``` + +The `-g` flag installs globally (user-level) and `-y` skips confirmation prompts. + +## Common Skill Categories + +When searching, consider these common categories: + +| Category | Example Queries | +| --------------- | ---------------------------------------- | +| Web Development | react, nextjs, typescript, css, tailwind | +| Testing | testing, jest, playwright, e2e | +| DevOps | deploy, docker, kubernetes, ci-cd | +| Documentation | docs, readme, changelog, api-docs | +| Code Quality | review, lint, refactor, best-practices | +| Design | ui, ux, design-system, accessibility | +| Productivity | workflow, automation, git | + +## Tips for Effective Searches + +1. **Use specific keywords**: "react testing" is better than just "testing" +2. **Try alternative terms**: If "deploy" doesn't work, try "deployment" or "ci-cd" +3. **Check popular sources**: Many skills come from `vercel-labs/agent-skills` or `ComposioHQ/awesome-claude-skills` + +## When No Skills Are Found + +If no relevant skills exist: + +1. Acknowledge that no existing skill was found +2. Offer to help with the task directly using your general capabilities +3. Suggest the user could create their own skill with `npx skills init` + +Example: + +``` +I searched for skills related to "xyz" but didn't find any matches. +I can still help you with this task directly! Would you like me to proceed? + +If this is something you do often, you could create your own skill: +npx skills init my-xyz-skill +``` diff --git a/skills/imagegen/LICENSE.txt b/skills/imagegen/LICENSE.txt new file mode 100644 index 0000000..13e25df --- /dev/null +++ b/skills/imagegen/LICENSE.txt @@ -0,0 +1,201 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf of + any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don\'t include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/skills/imagegen/SKILL.md b/skills/imagegen/SKILL.md new file mode 100644 index 0000000..4285e5e --- /dev/null +++ b/skills/imagegen/SKILL.md @@ -0,0 +1,356 @@ +--- +name: "imagegen" +description: "Generate or edit raster images when the task benefits from AI-created bitmap visuals such as photos, illustrations, textures, sprites, mockups, or transparent-background cutouts. Use when Codex should create a brand-new image, transform an existing image, or derive visual variants from references, and the output should be a bitmap asset rather than repo-native code or vector. Do not use when the task is better handled by editing existing SVG/vector/code-native assets, extending an established icon or logo system, or building the visual directly in HTML/CSS/canvas." +--- + +# Image Generation Skill + +Generates or edits images for the current project (for example website assets, game assets, UI mockups, product mockups, wireframes, logo design, photorealistic images, or infographics). + +## Top-level modes and rules + +This skill has exactly two top-level modes: + +- **Default built-in tool mode (preferred):** built-in `image_gen` tool for normal image generation, editing, and simple transparent-image requests. Does not require `OPENAI_API_KEY`. +- **Fallback CLI mode:** `scripts/image_gen.py` CLI. Use when the user explicitly asks for the CLI/API/model path, or after the user explicitly confirms a true model-native transparency fallback with `gpt-image-1.5`. Requires `OPENAI_API_KEY`. + +Within CLI fallback, the CLI exposes three subcommands: + +- `generate` +- `edit` +- `generate-batch` + +Rules: +- Use the built-in `image_gen` tool by default for normal image generation and editing requests. +- Do not switch to CLI fallback for ordinary quality, size, or file-path control. +- If the user explicitly asks for a transparent image/background, stay on built-in `image_gen` first: prompt for a flat removable chroma-key background, then remove it locally with the installed helper at `$CODEX_HOME/skills/.system/imagegen/scripts/remove_chroma_key.py`. +- Never silently switch from built-in `image_gen` or CLI `gpt-image-2` to CLI `gpt-image-1.5`. Treat this as a model/path downgrade and ask the user before doing it, unless the user has already explicitly requested `gpt-image-1.5`, `scripts/image_gen.py`, or CLI fallback. +- If a transparent request appears too complex for clean chroma-key removal, asks for true/native transparency, or local removal fails validation, explain that true transparency requires CLI `gpt-image-1.5 --background transparent --output-format png` because `gpt-image-2` does not support `background=transparent`, then ask whether to proceed. Run the CLI fallback only after the user confirms. +- The word `batch` by itself does not mean CLI fallback. If the user asks for many assets or says to batch-generate assets without explicitly asking for CLI/API/model controls, stay on the built-in path and issue one built-in call per requested asset or variant. +- If the built-in tool fails or is unavailable, tell the user the CLI fallback exists and that it requires `OPENAI_API_KEY`. Proceed only if the user explicitly asks for that fallback. +- If the user explicitly asks for CLI mode, use the bundled `scripts/image_gen.py` workflow. Do not create one-off SDK runners. +- Never modify `scripts/image_gen.py`. If something is missing, ask the user before doing anything else. + +Built-in save-path policy: +- In built-in tool mode, Codex saves generated images under `$CODEX_HOME/*` by default. +- Do not describe or rely on OS temp as the default built-in destination. +- Do not describe or rely on a destination-path argument (if any) on the built-in `image_gen` tool. If a specific location is needed, generate first and then move or copy the selected output from `$CODEX_HOME/generated_images/...`. +- Save-path precedence in built-in mode: + 1. If the user names a destination, move or copy the selected output there. + 2. If the image is meant for the current project, move or copy the final selected image into the workspace before finishing. + 3. If the image is only for preview or brainstorming, render it inline; the underlying file can remain at the default `$CODEX_HOME/*` path. +- Never leave a project-referenced asset only at the default `$CODEX_HOME/*` path. +- Do not overwrite an existing asset unless the user explicitly asked for replacement; otherwise create a sibling versioned filename such as `hero-v2.png` or `item-icon-edited.png`. + +Shared prompt guidance for both modes lives in `references/prompting.md` and `references/sample-prompts.md`. + +Fallback-only docs/resources for CLI mode: +- `references/cli.md` +- `references/image-api.md` +- `references/codex-network.md` +- `scripts/image_gen.py` + +Local post-processing helper: +- `$CODEX_HOME/skills/.system/imagegen/scripts/remove_chroma_key.py`: removes a flat chroma-key background from a generated image and writes a PNG/WebP with alpha. Prefer auto-key sampling, soft matte, and despill for antialiased edges. + +## When to use +- Generate a new image (concept art, product shot, cover, website hero) +- Generate a new image using one or more reference images for style, composition, or mood +- Edit an existing image (inpainting, lighting or weather transformations, background replacement, object removal, compositing, transparent background) +- Produce many assets or variants for one task + +## When not to use +- Extending or matching an existing SVG/vector icon set, logo system, or illustration library inside the repo +- Creating simple shapes, diagrams, wireframes, or icons that are better produced directly in SVG, HTML/CSS, or canvas +- Making a small project-local asset edit when the source file already exists in an editable native format +- Any task where the user clearly wants deterministic code-native output instead of a generated bitmap + +## Decision tree + +Think about two separate questions: + +1. **Intent:** is this a new image or an edit of an existing image? +2. **Execution strategy:** is this one asset or many assets/variants? + +Intent: +- If the user wants to modify an existing image while preserving parts of it, treat the request as **edit**. +- If the user provides images only as references for style, composition, mood, or subject guidance, treat the request as **generate**. +- If the user provides no images, treat the request as **generate**. + +Built-in edit semantics: +- Built-in edit mode is for images already visible in the conversation context, such as attached images or images generated earlier in the thread. +- If the user wants to edit a local image file with the built-in tool, first load it with built-in `view_image` tool so the image is visible in the conversation context, then proceed with the built-in edit flow. +- Do not promise arbitrary filesystem-path editing through the built-in tool. +- If a local file still needs direct file-path control, masks, or other explicit CLI-only parameters, use the explicit CLI fallback only when the user asks for it. +- For edits, preserve invariants aggressively and save non-destructively by default. + +Execution strategy: +- In the built-in default path, produce many assets or variants by issuing one `image_gen` call per requested asset or variant. +- In the CLI fallback path, use the CLI `generate-batch` subcommand only when the user explicitly chose CLI mode and needs many prompts/assets. +- For many distinct assets, do not use `n` as a substitute for separate prompts. `n` is for variants of one prompt; distinct assets need distinct built-in calls or distinct CLI `generate-batch` jobs. + +Assume the user wants a new image unless they clearly ask to change an existing one. + +## Workflow +1. Decide the top-level mode: built-in by default, including simple transparent-output requests; fallback CLI only if explicitly requested or after the user explicitly confirms a transparent-output fallback. +2. Decide the intent: `generate` or `edit`. +3. Decide whether the output is preview-only or meant to be consumed by the current project. +4. Decide the execution strategy: single asset vs repeated built-in calls vs CLI `generate-batch`. +5. Collect inputs up front: prompt(s), exact text (verbatim), constraints/avoid list, and any input images. +6. For every input image, label its role explicitly: + - reference image + - edit target + - supporting insert/style/compositing input +7. If the edit target is only on the local filesystem and you are staying on the built-in path, inspect it with `view_image` first so the image is available in conversation context. +8. If the user asked for a photo, illustration, sprite, product image, banner, or other explicitly raster-style asset, use `image_gen` rather than substituting SVG/HTML/CSS placeholders. If the request is for an icon, logo, or UI graphic that should match existing repo-native SVG/vector/code assets, prefer editing those directly instead. +9. Augment the prompt based on specificity: + - If the user's prompt is already specific and detailed, normalize it into a clear spec without adding creative requirements. + - If the user's prompt is generic, add tasteful augmentation only when it materially improves output quality. +10. Use the built-in `image_gen` tool by default. +11. For transparent-output requests, follow the transparent image guidance below: generate with built-in `image_gen` on a flat chroma-key background, copy the selected output into the workspace or `tmp/imagegen/`, run the installed `$CODEX_HOME/skills/.system/imagegen/scripts/remove_chroma_key.py` helper, and validate the alpha result before using it. If this path looks unsuitable or fails, ask before switching to CLI `gpt-image-1.5`. +12. Inspect outputs and validate: subject, style, composition, text accuracy, and invariants/avoid items. +13. Iterate with a single targeted change, then re-check. +14. For preview-only work, render the image inline; the underlying file may remain at the default `$CODEX_HOME/generated_images/...` path. +15. For project-bound work, move or copy the selected artifact into the workspace and update any consuming code or references. Never leave a project-referenced asset only at the default `$CODEX_HOME/generated_images/...` path. +16. For batches or multi-asset requests, persist every requested deliverable final in the workspace unless the user explicitly asked to keep outputs preview-only. Discarded variants do not need to be kept unless requested. +17. If the user explicitly chooses or confirms the CLI fallback, then use the fallback-only docs for model, quality, size, `input_fidelity`, masks, output format, output paths, and network setup. +18. Always report the final saved path(s) for any workspace-bound asset(s), plus the final prompt or prompt set and whether the built-in tool or fallback CLI mode was used. + +## Transparent image requests + +Transparent-image requests still use built-in `image_gen` first. Because the built-in tool does not expose a true transparent-background control, create a removable chroma-key source image and then convert the key color to alpha locally. + +Default sequence: +1. Use built-in `image_gen` to generate the requested subject on a perfectly flat solid chroma-key background. +2. Choose a key color that is unlikely to appear in the subject: default `#00ff00`, use `#ff00ff` for green subjects, and avoid `#0000ff` for blue subjects. +3. After generation, move or copy the selected source image from `$CODEX_HOME/generated_images/...` into the workspace or `tmp/imagegen/`. +4. Run the installed helper path, not a project-relative script path: + ```bash + python "${CODEX_HOME:-$HOME/.codex}/skills/.system/imagegen/scripts/remove_chroma_key.py" \ + --input \ + --out \ + --auto-key border \ + --soft-matte \ + --transparent-threshold 12 \ + --opaque-threshold 220 \ + --despill + ``` +5. Validate that the output has an alpha channel, transparent corners, plausible subject coverage, and no obvious key-color fringe. If a thin fringe remains, retry once with `--edge-contract 1`; use `--edge-feather 0.25` only when the edge is visibly stair-stepped and the subject is not shiny or reflective. +6. Save the final alpha PNG/WebP in the project if the asset is project-bound. Never leave a project-referenced transparent asset only under `$CODEX_HOME/*`. + +Prompt transparent requests like this: + +```text +Create the requested subject on a perfectly flat solid #00ff00 chroma-key background for background removal. +The background must be one uniform color with no shadows, gradients, texture, reflections, floor plane, or lighting variation. +Keep the subject fully separated from the background with crisp edges and generous padding. +Do not use #00ff00 anywhere in the subject. +No cast shadow, no contact shadow, no reflection, no watermark, and no text unless explicitly requested. +``` + +Do not automatically use CLI `gpt-image-1.5 --background transparent --output-format png` instead of chroma keying. Ask the user first when the user asks for true/native transparency, when local removal fails validation, or when the requested image is complex: hair, fur, feathers, smoke, glass, liquids, translucent materials, reflective objects, soft shadows, realistic product grounding, or subject colors that conflict with all practical key colors. + +Use a concise confirmation like: + +```text +This likely needs true native transparency. The default built-in path uses a chroma-key background plus local removal, but true transparency requires the CLI fallback with gpt-image-1.5 because gpt-image-2 does not support background=transparent. It also requires OPENAI_API_KEY. Should I proceed with that CLI fallback? +``` + +## Prompt augmentation + +Reformat user prompts into a structured, production-oriented spec. Make the user's goal clearer and more actionable, but do not blindly add detail. + +Treat this as prompt-shaping guidance, not a closed schema. Use only the lines that help, and add a short extra labeled line when it materially improves clarity. + +### Specificity policy + +Use the user's prompt specificity to decide how much augmentation is appropriate: + +- If the prompt is already specific and detailed, preserve that specificity and only normalize/structure it. +- If the prompt is generic, you may add tasteful augmentation when it will materially improve the result. + +Allowed augmentations: +- composition or framing hints +- polish level or intended-use hints +- practical layout guidance +- reasonable scene concreteness that supports the stated request + +Not allowed augmentations: +- extra characters or objects that are not implied by the request +- brand names, slogans, palettes, or narrative beats that are not implied +- arbitrary side-specific placement unless the surrounding layout supports it + +## Use-case taxonomy (exact slugs) + +Classify each request into one of these buckets and keep the slug consistent across prompts and references. + +Generate: +- photorealistic-natural — candid/editorial lifestyle scenes with real texture and natural lighting. +- product-mockup — product/packaging shots, catalog imagery, merch concepts. +- ui-mockup — app/web interface mockups and wireframes; specify the desired fidelity. +- infographic-diagram — diagrams/infographics with structured layout and text. +- scientific-educational — classroom explainers, scientific diagrams, and learning visuals with required labels and accuracy constraints. +- ads-marketing — campaign concepts and ad creatives with audience, brand position, scene, and exact tagline/copy. +- productivity-visual — slide, chart, workflow, and data-heavy business visuals. +- logo-brand — logo/mark exploration, vector-friendly. +- illustration-story — comics, children’s book art, narrative scenes. +- stylized-concept — style-driven concept art, 3D/stylized renders. +- historical-scene — period-accurate/world-knowledge scenes. + +Edit: +- text-localization — translate/replace in-image text, preserve layout. +- identity-preserve — try-on, person-in-scene; lock face/body/pose. +- precise-object-edit — remove/replace a specific element (including interior swaps). +- lighting-weather — time-of-day/season/atmosphere changes only. +- background-extraction — transparent background / clean cutout. Use built-in `image_gen` with chroma-key removal first for simple opaque subjects; ask before using CLI true transparency for complex subjects. +- style-transfer — apply reference style while changing subject/scene. +- compositing — multi-image insert/merge with matched lighting/perspective. +- sketch-to-render — drawing/line art to photoreal render. + +## Shared prompt schema + +Use the following labeled spec as shared prompt scaffolding for both top-level modes: + +```text +Use case: +Asset type: +Primary request: +Input images: (optional) +Scene/backdrop: +Subject:
+Style/medium: +Composition/framing: +Lighting/mood: +Color palette: +Materials/textures: +Text (verbatim): "" +Constraints: +Avoid: +``` + +Notes: +- `Asset type` and `Input images` are prompt scaffolding, not dedicated CLI flags. +- `Scene/backdrop` refers to the visual setting. It is not the same as the fallback CLI `background` parameter, which controls output transparency behavior. +- Fallback-only execution notes such as `Quality:`, `Input fidelity:`, masks, output format, and output paths belong in the CLI path only. Do not treat them as built-in `image_gen` tool arguments. + +Augmentation rules: +- Keep it short. +- Add only the details needed to improve the prompt materially. +- For edits, explicitly list invariants (`change only X; keep Y unchanged`). +- If any critical detail is missing and blocks success, ask a question; otherwise proceed. + +## Examples + +### Generation example (hero image) +```text +Use case: product-mockup +Asset type: landing page hero +Primary request: a minimal hero image of a ceramic coffee mug +Style/medium: clean product photography +Composition/framing: wide composition with usable negative space for page copy if needed +Lighting/mood: soft studio lighting +Constraints: no logos, no text, no watermark +``` + +### Edit example (invariants) +```text +Use case: precise-object-edit +Asset type: product photo background replacement +Primary request: replace only the background with a warm sunset gradient +Constraints: change only the background; keep the product and its edges unchanged; no text; no watermark +``` + +## Prompting best practices +- Structure prompt as scene/backdrop -> subject -> details -> constraints. +- Include intended use (ad, UI mock, infographic) to set the mode and polish level. +- Use camera/composition language for photorealism. +- Only use SVG/vector stand-ins when the user explicitly asked for vector output or a non-image placeholder. +- Quote exact text and specify typography + placement. +- For tricky words, spell them letter-by-letter and require verbatim rendering. +- For multi-image inputs, reference images by index and describe how they should be used. +- For edits, repeat invariants every iteration to reduce drift. +- Iterate with single-change follow-ups. +- If the prompt is generic, add only the extra detail that will materially help. +- If the prompt is already detailed, normalize it instead of expanding it. +- For CLI fallback only, see `references/cli.md` and `references/image-api.md` for model, `quality`, `input_fidelity`, masks, output format, and output-path guidance. +- For transparent images, use the built-in-first chroma-key workflow unless the request is complex enough to need true CLI transparency; ask before switching to CLI `gpt-image-1.5`. + +More principles shared by both modes: `references/prompting.md`. +Copy/paste specs shared by both modes: `references/sample-prompts.md`. + +## Guidance by asset type +Asset-type templates (website assets, game assets, wireframes, logo) are consolidated in `references/sample-prompts.md`. + +## gpt-image-2 guidance for CLI fallback + +The fallback CLI defaults to `gpt-image-2`. + +- Use `gpt-image-2` for new CLI/API workflows unless the request needs true model-native transparent output. +- If a transparent request may need CLI fallback, ask before using `gpt-image-1.5` unless the user already explicitly requested `gpt-image-1.5`, `scripts/image_gen.py`, or CLI fallback. Explain that the built-in chroma-key path is the default, but true transparency requires `gpt-image-1.5` because `gpt-image-2` does not support `background=transparent`. +- `gpt-image-2` always uses high fidelity for image inputs; do not set `input_fidelity` with this model. +- `gpt-image-2` supports `quality` values `low`, `medium`, `high`, and `auto`. +- Use `quality low` for fast drafts, thumbnails, and quick iterations. Use `medium`, `high`, or `auto` for final assets, dense text, diagrams, identity-sensitive edits, or high-resolution outputs. +- Square images are typically fastest to generate. Use `1024x1024` for fast square drafts. +- If the user asks for 4K-style output, use `3840x2160` for landscape or `2160x3840` for portrait. +- `gpt-image-2` size may be `auto` or `WIDTHxHEIGHT` if all constraints hold: max edge `<= 3840px`, both edges multiples of `16px`, long-to-short ratio `<= 3:1`, total pixels between `655,360` and `8,294,400`. + +Popular `gpt-image-2` sizes: +- `1024x1024` square +- `1536x1024` landscape +- `1024x1536` portrait +- `2048x2048` 2K square +- `2048x1152` 2K landscape +- `3840x2160` 4K landscape +- `2160x3840` 4K portrait +- `auto` + +## Fallback CLI mode only + +### Temp and output conventions +These conventions apply only to the CLI fallback. They do not describe built-in `image_gen` output behavior. +- Use `tmp/imagegen/` for intermediate files (for example JSONL batches); delete them when done. +- Write final artifacts under `output/imagegen/`. +- Use `--out` or `--out-dir` to control output paths; keep filenames stable and descriptive. + +### Dependencies +Prefer `uv` for dependency management in this repo. + +Required Python package: +```bash +uv pip install openai +``` + +Required for local chroma-key removal and optional downscaling: +```bash +uv pip install pillow +``` + +Portability note: +- If you are using the installed skill outside this repo, install dependencies into that environment with its package manager. +- In uv-managed environments, `uv pip install ...` remains the preferred path. + +### Environment +- `OPENAI_API_KEY` must be set for live API calls. +- Do not ask the user for `OPENAI_API_KEY` when using the built-in `image_gen` tool. +- Never ask the user to paste the full key in chat. Ask them to set it locally and confirm when ready. + +If the key is missing, give the user these steps: +1. Create an API key in the OpenAI platform UI: https://platform.openai.com/api-keys +2. Set `OPENAI_API_KEY` as an environment variable in their system. +3. Offer to guide them through setting the environment variable for their OS/shell if needed. + +If installation is not possible in this environment, tell the user which dependency is missing and how to install it into their active environment. + +### Script-mode notes +- CLI commands + examples: `references/cli.md` +- API parameter quick reference: `references/image-api.md` +- Network approvals / sandbox settings for CLI mode: `references/codex-network.md` + +## Reference map +- `references/prompting.md`: shared prompting principles for both modes. +- `references/sample-prompts.md`: shared copy/paste prompt recipes for both modes. +- `references/cli.md`: fallback-only CLI usage via `scripts/image_gen.py`. +- `references/image-api.md`: fallback-only API/CLI parameter reference. +- `references/codex-network.md`: fallback-only network/sandbox troubleshooting for CLI mode. +- `scripts/image_gen.py`: fallback-only CLI implementation. Do not load or use it unless the user explicitly chooses CLI mode or explicitly confirms a transparent request's true CLI transparency fallback. +- `$CODEX_HOME/skills/.system/imagegen/scripts/remove_chroma_key.py`: local post-processing helper for built-in transparent-image requests. diff --git a/skills/imagegen/agents/openai.yaml b/skills/imagegen/agents/openai.yaml new file mode 100644 index 0000000..5e01d44 --- /dev/null +++ b/skills/imagegen/agents/openai.yaml @@ -0,0 +1,6 @@ +interface: + display_name: "Image Gen" + short_description: "Generate or edit images for websites, games, and more" + icon_small: "./assets/imagegen-small.svg" + icon_large: "./assets/imagegen.png" + default_prompt: "Use $imagegen to make or edit an image for this project." diff --git a/skills/imagegen/assets/imagegen-small.svg b/skills/imagegen/assets/imagegen-small.svg new file mode 100644 index 0000000..20128b2 --- /dev/null +++ b/skills/imagegen/assets/imagegen-small.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/skills/imagegen/assets/imagegen.png b/skills/imagegen/assets/imagegen.png new file mode 100644 index 0000000..94b5454 Binary files /dev/null and b/skills/imagegen/assets/imagegen.png differ diff --git a/skills/imagegen/references/cli.md b/skills/imagegen/references/cli.md new file mode 100644 index 0000000..f4a5a63 --- /dev/null +++ b/skills/imagegen/references/cli.md @@ -0,0 +1,242 @@ +# CLI reference (`scripts/image_gen.py`) + +This file is for the fallback CLI mode only. Read it when the user explicitly asks to use `scripts/image_gen.py` / CLI / API / model controls, or after the user explicitly confirms that a transparent-output request should use the `gpt-image-1.5` true-transparency fallback path. + +`generate-batch` is a CLI subcommand in this fallback path. It is not a top-level mode of the skill. +The word `batch` in a user request is not CLI opt-in by itself. + +## What this CLI does +- `generate`: generate a new image from a prompt +- `edit`: edit one or more existing images +- `generate-batch`: run many generation jobs from a JSONL file after the user explicitly chooses CLI/API/model controls + +Real API calls require **network access** + `OPENAI_API_KEY`. `--dry-run` does not. + +## Quick start (works from any repo) +Set a stable path to the skill CLI (default `CODEX_HOME` is `~/.codex`): + +``` +export CODEX_HOME="${CODEX_HOME:-$HOME/.codex}" +export IMAGE_GEN="$CODEX_HOME/skills/.system/imagegen/scripts/image_gen.py" +``` + +Install dependencies into that environment with its package manager. In uv-managed environments, `uv pip install ...` remains the preferred path. + +## Quick start + +Dry-run (no API call; no network required; does not require the `openai` package): + +```bash +python "$IMAGE_GEN" generate \ + --prompt "Test" \ + --out output/imagegen/test.png \ + --dry-run +``` + +Notes: +- One-off dry-runs print the API payload and the computed output path(s). +- Repo-local finals should live under `output/imagegen/`. + +Generate (requires `OPENAI_API_KEY` + network): + +```bash +python "$IMAGE_GEN" generate \ + --prompt "A cozy alpine cabin at dawn" \ + --size 1024x1024 \ + --out output/imagegen/alpine-cabin.png +``` + +Edit: + +```bash +python "$IMAGE_GEN" edit \ + --image input.png \ + --prompt "Replace only the background with a warm sunset" \ + --out output/imagegen/sunset-edit.png +``` + +## Guardrails +- Use the bundled CLI directly (`python "$IMAGE_GEN" ...`) after activating the correct environment. +- Do **not** create one-off runners (for example `gen_images.py`) unless the user explicitly asks for a custom wrapper. +- **Never modify** `scripts/image_gen.py`. If something is missing, ask the user before doing anything else. +- Do not silently downgrade from CLI `gpt-image-2` or built-in `image_gen` to CLI `gpt-image-1.5`; ask first unless the user already explicitly requested `gpt-image-1.5`, `scripts/image_gen.py`, or CLI fallback. + +## Defaults +- Model: `gpt-image-2` +- Supported model family for this CLI: GPT Image models (`gpt-image-*`) +- Size: `auto` +- Quality: `medium` +- Output format: `png` +- Default one-off output path: `output/imagegen/output.png` +- Background: unspecified unless `--background` is set + +## gpt-image-2 size and model guidance + +`gpt-image-2` is the default model for new CLI fallback work. + +- Use `--quality low` for fast drafts, thumbnails, and quick iterations. +- Use `--quality medium`, `--quality high`, or `--quality auto` for final assets, dense text, diagrams, identity-sensitive edits, and high-resolution outputs. +- Square images are typically fastest. Use `--size 1024x1024` for quick square drafts. +- If the user asks for 4K-style output, use `--size 3840x2160` for landscape or `--size 2160x3840` for portrait. +- Do not pass `--input-fidelity` with `gpt-image-2`; this model always uses high fidelity for image inputs. +- Do not use `--background transparent` with `gpt-image-2`; the default transparent-image workflow uses built-in `image_gen` on a flat chroma-key background plus local removal. Use `gpt-image-1.5` only after the user explicitly confirms the true-transparent CLI fallback, unless they already requested `gpt-image-1.5`, `scripts/image_gen.py`, or CLI fallback. + +Popular `gpt-image-2` sizes: +- `1024x1024` +- `1536x1024` +- `1024x1536` +- `2048x2048` +- `2048x1152` +- `3840x2160` +- `2160x3840` +- `auto` + +`gpt-image-2` size constraints: +- max edge `<= 3840px` +- both edges multiples of `16px` +- long edge to short edge ratio `<= 3:1` +- total pixels between `655,360` and `8,294,400` +- outputs above `2560x1440` total pixels are experimental + +Fast draft: + +```bash +python "$IMAGE_GEN" generate \ + --prompt "A product thumbnail of a matte ceramic mug on a stone surface" \ + --quality low \ + --size 1024x1024 \ + --out output/imagegen/mug-draft.png +``` + +Final 2K landscape: + +```bash +python "$IMAGE_GEN" generate \ + --prompt "A polished landing-page hero image of a matte ceramic mug on a stone surface" \ + --quality high \ + --size 2048x1152 \ + --out output/imagegen/mug-hero.png +``` + +4K landscape: + +```bash +python "$IMAGE_GEN" generate \ + --prompt "A detailed architectural visualization at golden hour" \ + --size 3840x2160 \ + --quality high \ + --out output/imagegen/architecture-4k.png +``` + +True transparent fallback request: + +Ask for confirmation before using this command unless the user already explicitly requested `gpt-image-1.5`, `scripts/image_gen.py`, or CLI fallback. + +```bash +python "$IMAGE_GEN" generate \ + --model gpt-image-1.5 \ + --prompt "A clean product cutout on a transparent background" \ + --background transparent \ + --output-format png \ + --out output/imagegen/product-cutout.png +``` + +When using this path, explain briefly that built-in `image_gen` plus chroma-key removal is the default transparent-image path, but this request needs true model-native transparency. `gpt-image-2` does not support `background=transparent`, so `gpt-image-1.5` is required for this confirmed fallback. + +## Quality, input fidelity, and masks (CLI fallback only) +These are explicit CLI controls. They are not built-in `image_gen` tool arguments. + +- `--quality` works for `generate`, `edit`, and `generate-batch`: `low|medium|high|auto` +- `--input-fidelity` is **edit-only** and validated as `low|high`; it is not supported for `gpt-image-2` +- `--mask` is **edit-only** + +Example: + +```bash +python "$IMAGE_GEN" edit \ + --model gpt-image-1.5 \ + --image input.png \ + --prompt "Change only the background" \ + --quality high \ + --input-fidelity high \ + --out output/imagegen/background-edit.png +``` + +Mask notes: +- For multi-image edits, pass repeated `--image` flags. Their order is meaningful, so describe each image by index and role in the prompt. +- The CLI accepts a single `--mask`. +- Image and mask must be the same size and format and each under 50MB. +- Masks must include an alpha channel. +- If multiple input images are provided, the mask applies to the first image. +- Masking is prompt-guided; do not promise exact pixel-perfect mask boundaries. +- Use a PNG mask when possible; the script treats mask handling as best-effort and does not perform full preflight validation beyond file checks/warnings. +- In the edit prompt, repeat invariants (`change only the background; keep the subject unchanged`) to reduce drift. + +## Output handling +- Use `tmp/imagegen/` for temporary JSONL inputs or scratch files. +- Use `output/imagegen/` for final outputs. +- Reruns fail if a target file already exists unless you pass `--force`. +- `--out-dir` changes one-off naming to `image_1.`, `image_2.`, and so on. +- Downscaled copies use the default suffix `-web` unless you override it. + +## Common recipes + +Generate with augmentation fields: + +```bash +python "$IMAGE_GEN" generate \ + --prompt "A minimal hero image of a ceramic coffee mug" \ + --use-case "product-mockup" \ + --style "clean product photography" \ + --composition "wide product shot with usable negative space for page copy" \ + --constraints "no logos, no text" \ + --out output/imagegen/mug-hero.png +``` + +Generate + also write a downscaled copy for fast web loading: + +```bash +python "$IMAGE_GEN" generate \ + --prompt "A cozy alpine cabin at dawn" \ + --size 1024x1024 \ + --downscale-max-dim 1024 \ + --out output/imagegen/alpine-cabin.png +``` + +Generate multiple prompts concurrently (async batch): + +```bash +mkdir -p tmp/imagegen output/imagegen/batch +cat > tmp/imagegen/prompts.jsonl << 'EOF' +{"prompt":"Cavernous hangar interior with a compact shuttle parked near the center","use_case":"stylized-concept","composition":"wide-angle, low-angle","lighting":"volumetric light rays through drifting fog","constraints":"no logos or trademarks; no watermark","size":"1536x1024"} +{"prompt":"Gray wolf in profile in a snowy forest","use_case":"photorealistic-natural","composition":"eye-level","constraints":"no logos or trademarks; no watermark","size":"1024x1024"} +EOF + +python "$IMAGE_GEN" generate-batch \ + --input tmp/imagegen/prompts.jsonl \ + --out-dir output/imagegen/batch \ + --concurrency 5 + +rm -f tmp/imagegen/prompts.jsonl +``` + +Notes: +- `generate-batch` requires `--out-dir`. +- generate-batch requires --out-dir. +- Use `--concurrency` to control parallelism (default `5`). +- Per-job overrides are supported in JSONL (for example `size`, `quality`, `background`, `output_format`, `output_compression`, `moderation`, `n`, `model`, `out`, and prompt-augmentation fields). +- `--n` generates multiple variants for a single prompt; `generate-batch` is for many different prompts. +- In batch mode, per-job `out` is treated as a filename under `--out-dir`. +- For many requested deliverable assets, provide one prompt/job per distinct asset and use semantic filenames when possible. + +## CLI notes +- Supported sizes depend on the model. `gpt-image-2` supports flexible constrained sizes; older GPT Image models support `1024x1024`, `1536x1024`, `1024x1536`, or `auto`. +- True transparent CLI outputs require `output_format` to be `png` or `webp` and are not supported by `gpt-image-2`. +- `--prompt-file`, `--output-compression`, `--moderation`, `--max-attempts`, `--fail-fast`, `--force`, and `--no-augment` are supported. +- This CLI is intended for GPT Image models. Do not assume older non-GPT image-model behavior applies here. + +## See also +- API parameter quick reference for fallback CLI mode: `references/image-api.md` +- Prompt examples shared across both top-level modes: `references/sample-prompts.md` +- Network/sandbox notes for fallback CLI mode: `references/codex-network.md` +- Built-in-first transparent image workflow: `SKILL.md` and `$CODEX_HOME/skills/.system/imagegen/scripts/remove_chroma_key.py` diff --git a/skills/imagegen/references/codex-network.md b/skills/imagegen/references/codex-network.md new file mode 100644 index 0000000..5ce1fbc --- /dev/null +++ b/skills/imagegen/references/codex-network.md @@ -0,0 +1,33 @@ +# Codex network approvals / sandbox notes + +This file is for the fallback CLI mode only. Read it when the user explicitly asks to use `scripts/image_gen.py` / CLI / API / model controls, or after the user explicitly confirms that a transparent-output request should use the `gpt-image-1.5` true-transparency fallback path. + +This guidance is intentionally isolated from `SKILL.md` because it can vary by environment and may become stale. Prefer the defaults in your environment when in doubt. + +## Why am I asked to approve image generation calls? +The fallback CLI uses the OpenAI Image API, so it needs outbound network access. In many Codex setups, network access is disabled by default and/or the approval policy requires confirmation before networked commands run. + +## Important note about approvals vs network +- `--ask-for-approval never` suppresses approval prompts. +- It does **not** by itself enable network access. +- In `workspace-write`, network access still depends on your Codex configuration (for example `[sandbox_workspace_write] network_access = true`). + +## How do I reduce repeated approval prompts? +If you trust the repo and want fewer prompts, use a configuration or profile that both: +- enables network for the sandbox mode you plan to use +- sets an approval policy that matches your risk tolerance + +Example `~/.codex/config.toml` pattern: + +```toml +approval_policy = "on-request" +sandbox_mode = "workspace-write" + +[sandbox_workspace_write] +network_access = true +``` + +If you want quieter automation after network is enabled, you can choose a stricter approval policy, but do that intentionally and with care. + +## Safety note +Enabling network and reducing approvals lowers friction, but increases risk if you run untrusted code or work in an untrusted repository. diff --git a/skills/imagegen/references/image-api.md b/skills/imagegen/references/image-api.md new file mode 100644 index 0000000..db8567d --- /dev/null +++ b/skills/imagegen/references/image-api.md @@ -0,0 +1,90 @@ +# Image API quick reference + +This file is for the fallback CLI mode only. Use it when the user explicitly asks to use `scripts/image_gen.py` / CLI / API / model controls, or after the user explicitly confirms that a transparent-output request should use the `gpt-image-1.5` true-transparency fallback path. + +These parameters describe the Image API and bundled CLI fallback surface. Do not assume they are normal arguments on the built-in `image_gen` tool. + +## Scope +- This fallback CLI is intended for GPT Image models (`gpt-image-2`, `gpt-image-1.5`, `gpt-image-1`, and `gpt-image-1-mini`). +- The built-in `image_gen` tool and the fallback CLI do not expose the same controls. + +## Model summary + +| Model | Quality | Input fidelity | Resolutions | Recommended use | +| --- | --- | --- | --- | --- | +| `gpt-image-2` | `low`, `medium`, `high`, `auto` | Always high fidelity for image inputs; do not set `input_fidelity` | `auto` or flexible sizes that satisfy the constraints below | Default for new CLI/API workflows: high-quality generation and editing, text-heavy images, photorealism, compositing, identity-sensitive edits, and workflows where fewer retries matter | +| `gpt-image-1.5` | `low`, `medium`, `high`, `auto` | `low`, `high` | `1024x1024`, `1024x1536`, `1536x1024`, `auto` | True transparent-background fallback and backward-compatible workflows | +| `gpt-image-1` | `low`, `medium`, `high`, `auto` | `low`, `high` | `1024x1024`, `1024x1536`, `1536x1024`, `auto` | Legacy compatibility | +| `gpt-image-1-mini` | `low`, `medium`, `high`, `auto` | `low`, `high` | `1024x1024`, `1024x1536`, `1536x1024`, `auto` | Cost-sensitive draft batches and lower-stakes previews | + +## gpt-image-2 sizes + +`gpt-image-2` accepts `auto` or any `WIDTHxHEIGHT` size that satisfies all constraints: + +- Maximum edge length must be less than or equal to `3840px`. +- Both edges must be multiples of `16px`. +- Long edge to short edge ratio must not exceed `3:1`. +- Total pixels must be at least `655,360` and no more than `8,294,400`. + +Popular sizes: + +| Label | Size | Notes | +| --- | --- | --- | +| Square | `1024x1024` | Typical fast default | +| Landscape | `1536x1024` | Standard landscape | +| Portrait | `1024x1536` | Standard portrait | +| 2K square | `2048x2048` | Larger square output | +| 2K landscape | `2048x1152` | Widescreen output | +| 4K landscape | `3840x2160` | Widescreen 4K output | +| 4K portrait | `2160x3840` | Vertical 4K output | +| Auto | `auto` | Default size | + +Square images are typically fastest to generate. For 4K-style output, use `3840x2160` or `2160x3840`. + +## Endpoints +- Generate: `POST /v1/images/generations` (`client.images.generate(...)`) +- Edit: `POST /v1/images/edits` (`client.images.edit(...)`) + +## Core parameters for GPT Image models +- `prompt`: text prompt +- `model`: image model +- `n`: number of images (1-10) +- `size`: `auto` by default for `gpt-image-2`; flexible `WIDTHxHEIGHT` sizes are allowed only for `gpt-image-2`; older GPT Image models use `1024x1024`, `1536x1024`, `1024x1536`, or `auto` +- `quality`: `low`, `medium`, `high`, or `auto` +- `background`: output transparency behavior (`transparent`, `opaque`, or `auto`) for generated output; this is not the same thing as the prompt's visual scene/backdrop +- `output_format`: `png` (default), `jpeg`, `webp` +- `output_compression`: 0-100 (jpeg/webp only) +- `moderation`: `auto` (default) or `low` + +## Edit-specific parameters +- `image`: one or more input images. For GPT Image models, you can provide up to 16 images. +- `mask`: optional mask image +- `input_fidelity`: `low` or `high` only for models that support it; do not set this for `gpt-image-2` + +Model-specific note for `input_fidelity`: +- `gpt-image-2` always uses high fidelity for image inputs and does not support setting `input_fidelity`. +- `gpt-image-1` and `gpt-image-1-mini` preserve all input images, but the first image gets richer textures and finer details. +- `gpt-image-1.5` preserves the first 5 input images with higher fidelity. + +## Transparent backgrounds + +`gpt-image-2` does not currently support the Image API `background=transparent` parameter. The skill's default transparent-image path is built-in `image_gen` with a flat chroma-key background, followed by local alpha extraction with `python "${CODEX_HOME:-$HOME/.codex}/skills/.system/imagegen/scripts/remove_chroma_key.py"`. + +Use CLI `gpt-image-1.5` with `background=transparent` and a transparent-capable output format such as `png` or `webp` only after the user explicitly confirms that fallback, unless they already requested `gpt-image-1.5`, `scripts/image_gen.py`, or CLI fallback. If the user asks for true/native transparency, the subject is too complex for clean chroma-key removal, or local background removal fails validation, explain the tradeoff and ask before switching. + +## Output +- `data[]` list with `b64_json` per image +- The bundled `scripts/image_gen.py` CLI decodes `b64_json` and writes output files for you. + +## Limits and notes +- Input images and masks must be under 50MB. +- Use the edits endpoint when the user requests changes to an existing image. +- Masking is prompt-guided; exact shapes are not guaranteed. +- Large sizes and high quality increase latency and cost. +- Use `quality=low` for fast drafts, thumbnails, and quick iterations. Use `medium` or `high` for final assets, dense text, diagrams, identity-sensitive edits, or high-resolution outputs. +- High `input_fidelity` can materially increase input token usage on models that support it. +- If a request fails because a specific option is unsupported by the selected GPT Image model, retry manually without that option only when the option is not required by the user. If true transparent CLI output is required, ask before switching to `gpt-image-1.5` instead of dropping `background=transparent`, unless the user already explicitly chose that fallback. + +## Important boundary +- `quality`, `input_fidelity`, explicit masks, `background`, `output_format`, and related parameters are fallback-only execution controls. +- Do not assume they are built-in `image_gen` tool arguments. diff --git a/skills/imagegen/references/prompting.md b/skills/imagegen/references/prompting.md new file mode 100644 index 0000000..9d2da42 --- /dev/null +++ b/skills/imagegen/references/prompting.md @@ -0,0 +1,118 @@ +# Prompting best practices + +These prompting principles are shared by both top-level modes of the skill: +- built-in `image_gen` tool (default) +- explicit `scripts/image_gen.py` CLI fallback + +This file is about prompt structure, specificity, and iteration. Fallback-only execution controls such as `quality`, `input_fidelity`, masks, output format, and output paths live in the fallback docs. + +## Contents +- [Structure](#structure) +- [Specificity policy](#specificity-policy) +- [Allowed and disallowed augmentation](#allowed-and-disallowed-augmentation) +- [Composition and layout](#composition-and-layout) +- [Constraints and invariants](#constraints-and-invariants) +- [Text in images](#text-in-images) +- [Input images and references](#input-images-and-references) +- [Iterate deliberately](#iterate-deliberately) +- [Transparent images](#transparent-images) +- [Fallback-only execution controls](#fallback-only-execution-controls) +- [Use-case tips](#use-case-tips) +- [Where to find copy/paste recipes](#where-to-find-copypaste-recipes) + +## Structure +- Use a consistent order: scene/backdrop -> subject -> key details -> constraints -> output intent. +- Include intended use (ad, UI mock, infographic) to set the level of polish. +- For complex requests, use short labeled lines instead of one long paragraph. + +## Specificity policy +- If the user prompt is already specific and detailed, normalize it into a clean spec without adding creative requirements. +- If the prompt is generic, you may add tasteful detail when it materially improves the output. +- Treat examples in `sample-prompts.md` as fully-authored recipes, not as the default amount of augmentation to add to every request. +- For photorealism, include `photorealistic` directly when that is the goal, plus concrete real-world texture such as pores, wrinkles, fabric wear, material grain, or imperfect everyday detail. + +## Allowed and disallowed augmentation + +Allowed augmentation for generic prompts: +- composition and framing cues +- intended-use or polish-level hints +- practical layout guidance +- reasonable scene concreteness that supports the request + +Do not add: +- extra characters, props, or objects that are not implied +- brand palettes, slogans, or story beats that are not implied +- arbitrary side-specific placement unless the surrounding layout supports it + +## Composition and layout +- Specify framing and viewpoint (close-up, wide, top-down) and placement only when it materially helps. +- Call out negative space if the asset clearly needs room for UI or copy. +- Avoid making left/right layout decisions unless the user or surrounding layout supports them. +- For people, describe body framing, scale, gaze, and object interactions when they matter (`full body visible`, `looking down at the book`, `hands naturally gripping the handlebars`). + +## Constraints and invariants +- State what must not change (`keep background unchanged`). +- For edits, say `change only X; keep Y unchanged` and repeat invariants on every iteration to reduce drift. + +## Text in images +- Put literal text in quotes or ALL CAPS and specify typography (font style, size, color, placement). +- Spell uncommon words letter-by-letter if accuracy matters. +- For in-image copy, require verbatim rendering and no extra characters. +- In CLI fallback mode, use `medium` or `high` quality for small text, dense infographics, data-heavy slides, multi-font layouts, legends, axes, and footnotes. + +## Input images and references +- Do not assume that every provided image is an edit target. +- Label each image by index and role (`Image 1: edit target`, `Image 2: style reference`). +- If the user provides images for style, composition, or mood guidance and does not ask to modify them, treat the request as generation with references. +- If the user asks to preserve an existing image while changing specific parts, treat the request as an edit. +- For compositing, describe how the images interact (`place the subject from Image 2 into Image 1`). + +## Iterate deliberately +- Start with a clean base prompt, then make small single-change edits. +- Re-specify critical constraints when you iterate. +- Prefer one targeted follow-up at a time over rewriting the whole prompt. + +## Transparent images +- Use built-in `image_gen` first for transparent-image requests. If the subject is clearly too complex for chroma-key removal, explain the fallback and ask before switching to CLI. +- Prompt for a perfectly flat solid chroma-key background, usually `#00ff00`; use `#ff00ff` when the subject is green, and avoid key colors that appear in the subject. +- Explicitly prohibit shadows, gradients, floor planes, reflections, texture, and lighting variation in the background. +- Ask for crisp edges, generous padding, and no use of the key color inside the subject. +- After generation, remove the background locally with `python "${CODEX_HOME:-$HOME/.codex}/skills/.system/imagegen/scripts/remove_chroma_key.py" --input --out --auto-key border --soft-matte --transparent-threshold 12 --opaque-threshold 220 --despill` and validate the alpha result before shipping it. +- Use soft matte and despill for antialiased edges; hard tolerance-only removal is mainly for flat pixel-art or exact-color fixtures. +- Use CLI `gpt-image-1.5 --background transparent --output-format png` only after the user explicitly confirms the fallback, or when the user already explicitly requested `gpt-image-1.5`, `scripts/image_gen.py`, or CLI fallback. Ask first for true/native transparency requests, failed chroma-key validation, or complex transparent subjects such as hair, fur, glass, smoke, liquids, translucent materials, reflective objects, or soft shadows. + +## Fallback-only execution controls +- `quality`, `input_fidelity`, explicit masks, output format, and output paths are fallback-only execution controls. +- Do not assume they are built-in `image_gen` tool arguments. +- If the user explicitly chooses CLI fallback, see `references/cli.md` and `references/image-api.md` for those controls. +- In CLI fallback mode, `gpt-image-2` is the default. It supports `quality=low|medium|high|auto`; use `low` for fast drafts and thumbnails, and move to `medium`, `high`, or `auto` for final assets. +- `gpt-image-2` always uses high fidelity for image inputs, so do not set `input_fidelity` with that model. +- If a transparent request needs true CLI transparency, ask before using `gpt-image-1.5` unless the user already explicitly chose it. Explain that built-in chroma-key removal is the default path, but `gpt-image-2` does not support `background=transparent`. +- If the user asks for 4K-style output with `gpt-image-2`, use `3840x2160` for landscape or `2160x3840` for portrait. + +## Use-case tips +Generate: +- photorealistic-natural: Prompt as if a real photo is captured in the moment; use photography language (lens, lighting, framing); call for real texture; avoid over-stylized polish unless requested. +- product-mockup: Describe the product/packaging and materials; ensure clean silhouette and label clarity; if in-image text is needed, require verbatim rendering and specify typography. +- ui-mockup: Describe the target fidelity first (shippable mockup or low-fi wireframe), then focus on layout, hierarchy, and practical UI elements; avoid concept-art language. +- infographic-diagram: Define the audience and layout flow; label parts explicitly; require verbatim text; prefer higher quality in CLI mode for dense labels. +- logo-brand: Keep it simple and scalable; ask for a strong silhouette and balanced negative space; avoid decorative flourishes unless requested. +- ads-marketing: Write like a creative brief; include brand positioning, audience, desired vibe, scene, and exact tagline if text must appear. +- productivity-visual: Name the exact artifact (slide, chart, workflow diagram), define the canvas and hierarchy, provide real labels/data, and ask for readable typography and polished spacing. +- scientific-educational: Define audience, lesson objective, required labels, scientific constraints, arrows, and scan-friendly whitespace. +- illustration-story: Define panels or scene beats; keep each action concrete. +- stylized-concept: Specify style cues, material finish, and rendering approach (3D, painterly, clay) without inventing new story elements. +- historical-scene: State the location/date and required period accuracy; constrain clothing, props, and environment to match the era. + +Edit: +- text-localization: Change only the text; preserve layout, typography, spacing, and hierarchy; no extra words or reflow unless needed. +- identity-preserve: Lock identity (face, body, pose, hair, expression); change only the specified elements; match lighting and shadows. +- precise-object-edit: Specify exactly what to remove/replace; preserve surrounding texture and lighting; keep everything else unchanged. +- lighting-weather: Change only environmental conditions (light, shadows, atmosphere, precipitation); keep geometry, framing, and subject identity. +- background-extraction: For simple opaque subjects, request a clean cutout on a perfectly flat chroma-key background; crisp silhouette; generous padding; no shadows; no halos; preserve label text exactly; no restyling. Ask before using true CLI transparency for complex subjects. +- style-transfer: Specify style cues to preserve (palette, texture, brushwork) and what must change; add `no extra elements` to prevent drift. +- compositing: Reference inputs by index; specify what moves where; match lighting, perspective, and scale; keep the base framing unchanged. +- sketch-to-render: Preserve layout, proportions, and perspective; choose materials and lighting that support the supplied sketch without adding new elements. + +## Where to find copy/paste recipes +For copy/paste prompt specs (examples only), see `references/sample-prompts.md`. This file focuses on principles, specificity, and iteration patterns. diff --git a/skills/imagegen/references/sample-prompts.md b/skills/imagegen/references/sample-prompts.md new file mode 100644 index 0000000..d949295 --- /dev/null +++ b/skills/imagegen/references/sample-prompts.md @@ -0,0 +1,433 @@ +# Sample prompts (copy/paste) + +These prompt recipes are shared across both top-level modes of the skill: +- built-in `image_gen` tool (default) +- `scripts/image_gen.py` CLI fallback for explicit CLI/API/model requests or user-confirmed true-transparent-output fallback requests + +Use these as starting points. They are intentionally complete prompt recipes, not the default amount of augmentation to add to every user request. + +When adapting a user's prompt: +- keep user-provided requirements +- only add detail according to the specificity policy in `SKILL.md` +- do not treat every example below as permission to invent extra story elements + +The labeled lines are prompt scaffolding, not a closed schema. `Asset type` and `Input images` are prompt-only scaffolding; the CLI does not expose them as dedicated flags. + +Execution details such as explicit CLI flags, `quality`, `input_fidelity`, masks, output formats, and local output paths depend on mode. Use the built-in tool by default, including simple transparent-image requests. For transparent images, prompt for a flat chroma-key background and remove it locally with `python "${CODEX_HOME:-$HOME/.codex}/skills/.system/imagegen/scripts/remove_chroma_key.py"`; only apply CLI-specific controls when the user explicitly opts into fallback mode or explicitly confirms that the transparent request should use true CLI transparency. + +CLI model notes: +- `gpt-image-2` is the fallback CLI default for new workflows. +- `gpt-image-2` supports `quality` values `low`, `medium`, `high`, and `auto`. +- For 4K-style `gpt-image-2` output, use `3840x2160` or `2160x3840`. +- If transparent output needs true CLI fallback, ask before using `gpt-image-1.5` unless the user already explicitly requested `gpt-image-1.5`, `scripts/image_gen.py`, or CLI fallback. Explain that built-in chroma-key removal is the default path, but `gpt-image-2` does not support `background=transparent`. +- Do not set `input_fidelity` with `gpt-image-2`; image inputs already use high fidelity. + +For prompting principles (structure, specificity, invariants, iteration), see `references/prompting.md`. + +## Generate + +### photorealistic-natural +``` +Use case: photorealistic-natural +Primary request: candid photo of an elderly sailor on a small fishing boat adjusting a net +Scene/backdrop: coastal water with soft haze +Subject: weathered skin with wrinkles and sun texture +Style/medium: photorealistic candid photo +Composition/framing: medium close-up, eye-level +Lighting/mood: soft coastal daylight, shallow depth of field, subtle film grain +Materials/textures: real skin texture, worn fabric, salt-worn wood +Constraints: natural color balance; no heavy retouching; no glamorization; no watermark +Avoid: studio polish; staged look +``` + +### product-mockup +``` +Use case: product-mockup +Primary request: premium product photo of a matte black shampoo bottle with a minimal label +Scene/backdrop: clean studio gradient from light gray to white +Subject: single bottle centered with subtle reflection +Style/medium: premium product photography +Composition/framing: centered, slight three-quarter angle, generous padding +Lighting/mood: softbox lighting, clean highlights, controlled shadows +Materials/textures: matte plastic, crisp label printing +Constraints: no logos or trademarks; no watermark +``` + +### ui-mockup +``` +Use case: ui-mockup +Primary request: mobile app home screen for a local farmers market with vendors and daily specials +Asset type: mobile app screen +Style/medium: realistic product UI, not concept art +Composition/framing: clean vertical mobile layout with clear hierarchy +Constraints: practical layout, clear typography, no logos or trademarks, no watermark +``` + +### infographic-diagram +``` +Use case: infographic-diagram +Primary request: detailed infographic of an automatic coffee machine flow +Scene/backdrop: clean, light neutral background +Subject: bean hopper -> grinder -> brew group -> boiler -> water tank -> drip tray +Style/medium: clean vector-like infographic with clear callouts and arrows +Composition/framing: vertical poster layout, top-to-bottom flow +Text (verbatim): "Bean Hopper", "Grinder", "Brew Group", "Boiler", "Water Tank", "Drip Tray" +Constraints: clear labels, strong contrast, no logos or trademarks, no watermark +``` + +### scientific-educational +``` +Use case: scientific-educational +Primary request: biology diagram titled "Cellular Respiration at a Glance" for high school students +Scene/backdrop: clean white classroom handout background +Subject: glucose turns into energy inside a cell; include glycolysis, Krebs cycle, and electron transport chain +Style/medium: flat scientific diagram with consistent icons, arrows, and readable labels +Composition/framing: landscape slide-style layout with clear hierarchy and generous whitespace +Text (verbatim): "Cellular Respiration at a Glance", "Glucose", "Pyruvate", "ATP", "NADH", "FADH2", "CO2", "O2", "H2O" +Constraints: scientifically plausible; avoid tiny text; no extra decoration; no watermark +``` + +### logo-brand +``` +Use case: logo-brand +Primary request: original logo for "Field & Flour", a local bakery +Style/medium: vector logo mark; flat colors; minimal +Composition/framing: single centered logo on a plain background with generous padding +Constraints: strong silhouette, balanced negative space; original design only; no gradients unless essential; no trademarks; no watermark +``` + +### illustration-story +``` +Use case: illustration-story +Primary request: 4-panel comic about a pet left alone at home +Scene/backdrop: cozy living room across panels +Subject: pet reacting to the owner leaving, then relaxing, then returning to a composed pose +Style/medium: comic illustration with clear panels +Composition/framing: 4 equal-sized vertical panels, readable actions per panel +Constraints: no text; no logos or trademarks; no watermark +``` + +### stylized-concept +``` +Use case: stylized-concept +Primary request: cavernous hangar interior with tall support beams and drifting fog +Scene/backdrop: industrial hangar interior, deep scale, light haze +Subject: compact shuttle parked near the center +Style/medium: cinematic concept art, industrial realism +Composition/framing: wide-angle, low-angle +Lighting/mood: volumetric light rays cutting through fog +Constraints: no logos or trademarks; no watermark +``` + +### ads-marketing +``` +Use case: ads-marketing +Primary request: campaign image for a streetwear brand called Thread +Subject: group of friends hanging out together in a stylish urban setting +Style/medium: polished youth streetwear campaign photography +Composition/framing: vertical ad layout with natural poses and integrated headline space +Lighting/mood: contemporary, energetic, tasteful +Text (verbatim): "Yours to Create." +Constraints: render the tagline exactly once; clean legible typography; no extra text; no watermarks; no unrelated logos +``` + +### productivity-visual +``` +Use case: productivity-visual +Primary request: one pitch-deck slide titled "Market Opportunity" +Asset type: fundraising slide image +Style/medium: clean modern deck slide, white background, crisp sans-serif typography +Subject: TAM/SAM/SOM concentric-circle diagram plus a small growth bar chart from 2021 to 2026 +Composition/framing: 16:9 landscape slide, clear data hierarchy, polished spacing +Text (verbatim): "Market Opportunity", "TAM: $42B", "SAM: $8.7B", "SOM: $340M", "AGI Research, 2024", "Internal analysis" +Constraints: readable labels, no clip art, no stock photography, no decorative clutter, no watermark +``` + +### historical-scene +``` +Use case: historical-scene +Primary request: outdoor crowd scene in Bethel, New York on August 16, 1969 +Scene/backdrop: open field with period-appropriate staging +Subject: crowd in period-accurate clothing, authentic environment +Style/medium: photorealistic photo +Composition/framing: wide shot, eye-level +Constraints: period-accurate details; no modern objects; no logos or trademarks; no watermark +``` + +## Asset type templates (taxonomy-aligned) + +### Website assets template +``` +Use case: +Asset type: +Primary request: +Scene/backdrop: +Subject:
+Style/medium: +Composition/framing: +Lighting/mood: +Color palette: +Constraints: +``` + +### Website assets example: minimal hero background +``` +Use case: stylized-concept +Asset type: landing page hero background +Primary request: minimal abstract background with a soft gradient and subtle texture +Style/medium: matte illustration / soft-rendered abstract background +Composition/framing: wide composition with usable negative space for page copy +Lighting/mood: gentle studio glow +Color palette: restrained neutral palette +Constraints: no text; no logos; no watermark +``` + +### Website assets example: feature section illustration +``` +Use case: stylized-concept +Asset type: feature section illustration +Primary request: simple abstract shapes suggesting connection and flow +Scene/backdrop: subtle light-gray backdrop with faint texture +Style/medium: flat illustration; soft shadows; restrained contrast +Composition/framing: centered cluster; open margins for UI +Color palette: muted neutral palette +Constraints: no text; no logos; no watermark +``` + +### Website assets example: blog header image +``` +Use case: photorealistic-natural +Asset type: blog header image +Primary request: overhead desk scene with notebook, pen, and coffee cup +Scene/backdrop: warm wooden tabletop +Style/medium: photorealistic photo +Composition/framing: wide crop with clean room for page copy +Lighting/mood: soft morning light +Constraints: no text; no logos; no watermark +``` + +### Game assets template +``` +Use case: stylized-concept +Asset type: +Primary request: +Scene/backdrop: (if applicable) +Subject:
+Style/medium: ; +Composition/framing: ; ; +Lighting/mood: