From 78f0cf2481acc0b12473de45bc9d3c209dd43b48 Mon Sep 17 00:00:00 2001 From: CyberSecurityErial <2710555967@qq.com> Date: Sun, 19 Jul 2026 08:29:04 +0800 Subject: [PATCH 1/8] feat(kernels): add semantic operator catalog --- rl_engine/kernels/registry.py | 138 ++++- rl_engine/kernels/semantic_registry.py | 770 +++++++++++++++++++++++++ 2 files changed, 876 insertions(+), 32 deletions(-) create mode 100644 rl_engine/kernels/semantic_registry.py diff --git a/rl_engine/kernels/registry.py b/rl_engine/kernels/registry.py index 041ed3e1..a9058782 100644 --- a/rl_engine/kernels/registry.py +++ b/rl_engine/kernels/registry.py @@ -1,11 +1,19 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2026 RL-Kernel Contributors +from __future__ import annotations + import importlib import os from enum import Enum, EnumMeta from typing import Any, Dict, Optional, Set, Type +from rl_engine.kernels.semantic_registry import ( + OperatorBackendDescriptor, + OperatorFallbackPolicy, + OperatorLifecycle, + SemanticOperatorCatalog, +) from rl_engine.platforms.device import device_ctx from rl_engine.utils.logger import logger @@ -16,9 +24,11 @@ class _KernelEnumMeta(EnumMeta): def __getitem__(cls, name: str): try: return super().__getitem__(name) - except KeyError as e: + except KeyError as exc: valid_ops = ", ".join(cls.__members__.keys()) - raise ValueError(f"Operator '{name}' not found. Supported backends: {valid_ops}") from e + raise ValueError( + f"Operator '{name}' not found. Supported backends: {valid_ops}" + ) from exc class OpBackend(Enum, metaclass=_KernelEnumMeta): @@ -78,6 +88,52 @@ class OpBackend(Enum, metaclass=_KernelEnumMeta): PYTORCH_NATIVE_EMBEDDING = "rl_engine.kernels.ops.pytorch.linear.embedding.NativeEmbeddingOp" +def _default_semantic_descriptors() -> tuple[OperatorBackendDescriptor, ...]: + return ( + OperatorBackendDescriptor( + semantic_op="selected_logprob", + backend_id="rlkernel.reference_logp", + supported_targets=frozenset({"rollout", "training"}), + supported_devices=frozenset({"cpu", "cuda", "rocm"}), + supported_dtypes=frozenset({"float32", "bfloat16", "float16"}), + supported_topologies={ + "world_size": (1,), + "tensor_parallel_size": (1,), + "context_parallel_size": (1,), + "sharding": ("unsharded",), + }, + determinism_or_alignment_properties={ + "algorithm": "pytorch.log_softmax_gather", + "batch_invariant": True, + "deterministic": True, + "strict_observable": True, + }, + lifecycle=OperatorLifecycle.ENGINE_CONSTRUCTION, + implementation_class_or_factory=( + "rl_engine.kernels.ops.pytorch.loss.logp.NativeLogpOp" + ), + fallback_policy=OperatorFallbackPolicy.ERROR, + version_or_build_fingerprint="NativeLogpOp-selected-logprob-v1", + ), + OperatorBackendDescriptor( + semantic_op="selected_logprob", + backend_id="native", + supported_targets=frozenset({"rollout", "training"}), + supported_devices=frozenset({"*"}), + supported_dtypes=frozenset({"*"}), + supported_topologies={"*": "*"}, + determinism_or_alignment_properties={ + "selection": "runtime_native", + "strict_observable": False, + }, + lifecycle=OperatorLifecycle.ENGINE_CONSTRUCTION, + implementation_class_or_factory=None, + fallback_policy=OperatorFallbackPolicy.RUNTIME_MANAGED, + version_or_build_fingerprint="runtime-native-unresolved-v1", + ), + ) + + def resolve_logp_op_type( logp_backend: Optional[str] = None, *, @@ -126,14 +182,12 @@ def resolve_logp_op_type( class KernelRegistry: - """ - Central dispatcher for high-performance kernels. - Handles dynamic routing between ROCm and CUDA backends at runtime. - """ + """Legacy hardware dispatcher plus a composed semantic operator catalog.""" def __init__(self): self._instance_cache: Dict[str, Any] = {} self._failed_backends: Set[str] = set() + self.semantic = SemanticOperatorCatalog(_default_semantic_descriptors()) self._priority_map = { "cuda": { @@ -163,23 +217,33 @@ def __init__(self): OpBackend.CUDA_DETERMINISTIC_LOGP, OpBackend.PYTORCH_NATIVE, ], - "attn": [OpBackend.FLASH_ATTN, OpBackend.TRITON_GENERIC, OpBackend.PYTORCH_ATTN], + "attn": [ + OpBackend.FLASH_ATTN, + OpBackend.TRITON_GENERIC, + OpBackend.PYTORCH_ATTN, + ], "attention": [OpBackend.PYTORCH_NATIVE_ATTENTION], "kv_cache_attention": [OpBackend.PYTORCH_NATIVE_KV_CACHE_ATTN], "grpo_loss": [OpBackend.TRITON_GRPO_LOSS, OpBackend.PYTORCH_GRPO_LOSS], - "linear_logp": [OpBackend.TRITON_LINEAR_LOGP, OpBackend.PYTORCH_LINEAR_LOGP], + "linear_logp": [ + OpBackend.TRITON_LINEAR_LOGP, + OpBackend.PYTORCH_LINEAR_LOGP, + ], "ratio_kl": [OpBackend.TRITON_RATIO_KL, OpBackend.PYTORCH_RATIO_KL], "rms_norm": [OpBackend.PYTORCH_NATIVE_RMS_NORM], "lm_head": [OpBackend.PYTORCH_NATIVE_LM_HEAD], "embedding": [OpBackend.PYTORCH_NATIVE_EMBEDDING], "silu": [OpBackend.PYTORCH_NATIVE_SILU], "swiglu": [OpBackend.PYTORCH_NATIVE_SWIGLU], - # Default dispatch logic for new operators "matmul": [OpBackend.PYTORCH_NATIVE_MATMUL], "rope": [OpBackend.PYTORCH_NATIVE_ROPE], }, "rocm": { - "logp": [OpBackend.ROCM_AITER, OpBackend.TRITON_GENERIC, OpBackend.PYTORCH_NATIVE], + "logp": [ + OpBackend.ROCM_AITER, + OpBackend.TRITON_GENERIC, + OpBackend.PYTORCH_NATIVE, + ], "logp_deterministic": [OpBackend.PYTORCH_NATIVE], "logp_deterministic_indexed": [OpBackend.PYTORCH_NATIVE], "attn": [ @@ -191,7 +255,10 @@ def __init__(self): "kv_cache_attention": [OpBackend.PYTORCH_NATIVE_KV_CACHE_ATTN], "grpo_loss": [OpBackend.TRITON_GRPO_LOSS, OpBackend.PYTORCH_GRPO_LOSS], "rope": [OpBackend.PYTORCH_NATIVE_ROPE], - "linear_logp": [OpBackend.TRITON_LINEAR_LOGP, OpBackend.PYTORCH_LINEAR_LOGP], + "linear_logp": [ + OpBackend.TRITON_LINEAR_LOGP, + OpBackend.PYTORCH_LINEAR_LOGP, + ], "ratio_kl": [OpBackend.TRITON_RATIO_KL, OpBackend.PYTORCH_RATIO_KL], "matmul": [OpBackend.PYTORCH_NATIVE_MATMUL], "rms_norm": [OpBackend.PYTORCH_NATIVE_RMS_NORM], @@ -237,14 +304,15 @@ def _adjust_priority_from_env(self): OpBackend.ROCM_FLASH_ATTN, OpBackend.TRITON_GENERIC, ] - elif rocm_attn_backend and rocm_attn_backend not in {"native", "pytorch", "sdpa"}: + elif rocm_attn_backend: logger.warning( "Unknown RL_KERNEL_ROCM_ATTN_BACKEND=%s; using default ROCm attention priority.", rocm_attn_backend, ) def _adjust_priority_for_hardware(self): - """Adjust CUDA priorities for hardware-gated experimental and production kernels.""" + """Adjust CUDA priorities for hardware-gated kernels.""" + if device_ctx.device_type != "cuda": return try: @@ -266,8 +334,6 @@ def _adjust_priority_for_hardware(self): if OpBackend.CUDA_FUSED_LOGP_SM90 not in logp_list: logp_list.insert(0, OpBackend.CUDA_FUSED_LOGP_SM90) - # The fused linear-logp SM90 kernel uses TMA bulk-tensor copies built - # for sm_90a -- gate strictly on cc_major == 9 (Hopper), not >= 9. linear_logp_compiled = _EXT_AVAILABLE and hasattr(_C, "fused_linear_logp_sm90") if linear_logp_compiled and cc_major == 9: ll_list = self._priority_map["cuda"]["linear_logp"] @@ -278,25 +344,26 @@ def _adjust_priority_for_hardware(self): f"SM{cc}: fused linear-logp SM90 kernel not compiled into _C; " "using generic linear-logp backend." ) - except Exception as e: - logger.warning(f"Failed to probe device capability: {e}") + except Exception as exc: + logger.warning(f"Failed to probe device capability: {exc}") def get_op(self, op_type: str) -> Any: - """Core distribution logic: Automatically select the best operator - based on hardware and priority. - """ + """Select the best legacy operator based on hardware and priority.""" + if device_ctx.is_rocm: platform = "rocm" elif device_ctx.device_type == "cuda": platform = "cuda" else: platform = "cpu" - candidates = self._priority_map.get(platform, {}).get(op_type, [OpBackend.PYTORCH_NATIVE]) + candidates = self._priority_map.get(platform, {}).get( + op_type, + [OpBackend.PYTORCH_NATIVE], + ) for backend in candidates: if backend.name in self._instance_cache: return self._instance_cache[backend.name] - if backend.name in self._failed_backends: continue @@ -306,8 +373,8 @@ def get_op(self, op_type: str) -> Any: op_instance = op_class() self._instance_cache[backend.name] = op_instance return op_instance - except Exception as e: - logger.error(f"Failed to instantiate {backend.name}: {e}") + except Exception as exc: + logger.error(f"Failed to instantiate {backend.name}: {exc}") self._failed_backends.add(backend.name) else: self._failed_backends.add(backend.name) @@ -315,23 +382,30 @@ def get_op(self, op_type: str) -> Any: raise RuntimeError(f"No functional backend found for {op_type} on {platform}") def _load_backend(self, backend: OpBackend) -> Optional[Type]: - """Dynamic loading technique: Import modules only when needed - and check environment dependencies. - """ + """Import a legacy backend and distinguish wrapper bugs from absence.""" + module_path, class_name = backend.value.rsplit(".", 1) try: module = importlib.import_module(module_path) return getattr(module, class_name) - except (ImportError, AttributeError, ModuleNotFoundError) as e: - missing_module = str(e.name) if hasattr(e, "name") else "" + except (ImportError, AttributeError, ModuleNotFoundError) as exc: + missing_module = str(exc.name) if hasattr(exc, "name") else "" is_missing_backend = missing_module and ( missing_module == module_path or module_path.startswith(missing_module) ) if missing_module and "rl_engine" in missing_module and not is_missing_backend: - logger.critical(f"Internal wrapper implementation bug in '{module_path}': {e}") - raise e - logger.warning(f"Backend {backend.name} unavailable: {e}. Falling back...") + logger.critical(f"Internal wrapper implementation bug in '{module_path}': {exc}") + raise + logger.warning(f"Backend {backend.name} unavailable: {exc}. Falling back...") return None kernel_registry = KernelRegistry() + + +__all__ = [ + "KernelRegistry", + "OpBackend", + "kernel_registry", + "resolve_logp_op_type", +] diff --git a/rl_engine/kernels/semantic_registry.py b/rl_engine/kernels/semantic_registry.py new file mode 100644 index 00000000..b4ee25bb --- /dev/null +++ b/rl_engine/kernels/semantic_registry.py @@ -0,0 +1,770 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Exact semantic-operator catalog with case-local instantiation state.""" + +from __future__ import annotations + +import hashlib +import importlib +import inspect +import json +from dataclasses import dataclass, field, fields, replace +from enum import Enum +from pathlib import Path +from types import CodeType, MappingProxyType +from typing import Any, Callable, Iterable, Mapping, Optional, Sequence, cast + + +class OperatorLifecycle(str, Enum): + REQUEST = "request" + ENGINE_CONSTRUCTION = "engine_construction" + DISTRIBUTED_CONTEXT = "distributed_context" + PROCESS = "process" + + +class OperatorFallbackPolicy(str, Enum): + ERROR = "error" + DECLARED = "declared" + RUNTIME_MANAGED = "runtime_managed" + + +@dataclass(frozen=True) +class OperatorResolutionPolicy: + strict: bool = True + allow_test_backends: bool = False + + +_Policy = Optional[OperatorResolutionPolicy] + + +class _JsonRecord: + def to_dict(self) -> dict[str, Any]: + return { + item.name: _json_value(getattr(self, item.name)) for item in fields(cast(Any, self)) + } + + +@dataclass(frozen=True) +class OperatorRequirements(_JsonRecord): + device: str + dtype: str + topology: Mapping[str, Any] = field(default_factory=dict) + alignment_properties: Mapping[str, Any] = field(default_factory=dict) + schema_version: str = "rlkernel.semantic_operator.requirements.v1" + + def __post_init__(self) -> None: + normalized = ( + ("device", _normalize_device(self.device)), + ("dtype", _normalize_dtype(self.dtype)), + ("topology", _freeze(self.topology)), + ("alignment_properties", _freeze(self.alignment_properties)), + ) + for name, value in normalized: + object.__setattr__(self, name, value) + + +@dataclass(frozen=True) +class OperatorBackendDescriptor(_JsonRecord): + semantic_op: str + backend_id: str + supported_targets: frozenset[str] + supported_devices: frozenset[str] + supported_dtypes: frozenset[str] + supported_topologies: Mapping[str, Any] + determinism_or_alignment_properties: Mapping[str, Any] + lifecycle: OperatorLifecycle + implementation_class_or_factory: Optional[str | Callable[..., Any]] + fallback_policy: OperatorFallbackPolicy + version_or_build_fingerprint: str + is_smoke_only: bool = False + schema_version: str = "rlkernel.semantic_operator.backend_descriptor.v1" + + def __post_init__(self) -> None: + values = { + "semantic_op": self.semantic_op.strip(), + "backend_id": self.backend_id.strip(), + "supported_targets": _normalized_values(self.supported_targets, str), + "supported_devices": _normalized_values(self.supported_devices, _normalize_device), + "supported_dtypes": _normalized_values(self.supported_dtypes, _normalize_dtype), + } + for name, value in values.items(): + if not value: + raise ValueError(f"{name} must not be empty") + if not self.version_or_build_fingerprint.strip(): + raise ValueError("version_or_build_fingerprint must not be empty") + values.update( + supported_topologies=_freeze(self.supported_topologies), + determinism_or_alignment_properties=_freeze(self.determinism_or_alignment_properties), + lifecycle=OperatorLifecycle(self.lifecycle), + fallback_policy=OperatorFallbackPolicy(self.fallback_policy), + ) + for name, value in values.items(): + object.__setattr__(self, name, value) + + @property + def implementation_reference(self) -> Optional[str]: + return _reference(self.implementation_class_or_factory) + + @property + def is_strictly_observable(self) -> bool: + return bool( + self.determinism_or_alignment_properties.get( + "strict_observable", self.implementation_class_or_factory is not None + ) + ) + + @property + def descriptor_fingerprint(self) -> str: + return _fingerprint(self.to_dict(include_descriptor_fingerprint=False)) + + def to_dict(self, *, include_descriptor_fingerprint: bool = True) -> dict[str, Any]: + result = super().to_dict() + result["implementation_class_or_factory"] = self.implementation_reference + if include_descriptor_fingerprint: + result["descriptor_fingerprint"] = self.descriptor_fingerprint + return result + + +@dataclass(frozen=True) +class OperatorCapabilityDecision(_JsonRecord): + capability: str + requested: Any + supported: Any + passed: bool + reason: str + + +@dataclass(frozen=True) +class OperatorResolutionTrace(_JsonRecord): + semantic_op: str + requested_backend: str + target: str + strict: bool + status: str + concrete_backend: Optional[str] + implementation_reference: Optional[str] + descriptor_fingerprint: Optional[str] + capability_decisions: tuple[OperatorCapabilityDecision, ...] + fallback_attempts: tuple[str, ...] = () + schema_version: str = "rlkernel.semantic_operator.resolution_trace.v1" + + +@dataclass(frozen=True) +class OperatorResolution(_JsonRecord): + descriptor: OperatorBackendDescriptor + requirements: OperatorRequirements + target: str + strict: bool + trace: OperatorResolutionTrace + schema_version: str = "rlkernel.semantic_operator.resolution.v1" + + +@dataclass(frozen=True) +class OperatorInstanceProvenance(_JsonRecord): + semantic_op: str + backend_id: str + target: str + factory_reference: str + concrete_implementation: str + descriptor_fingerprint: str + implementation_fingerprint: str + instance_fingerprint: str + factory_options: Mapping[str, Any] = field(default_factory=dict) + factory_options_fingerprint: str = "" + schema_version: str = "rlkernel.semantic_operator.instance_provenance.v1" + + def __post_init__(self) -> None: + object.__setattr__(self, "factory_options", _freeze(self.factory_options)) + + +@dataclass(frozen=True) +class _InstanceRecord: + instance: Any + descriptor_fingerprint: str + target: str + factory: Callable[..., Any] + factory_options: Mapping[str, Any] + + +class OperatorRegistrationError(ValueError): + pass + + +class OperatorResolutionError(RuntimeError): + def __init__(self, message: str, trace: OperatorResolutionTrace): + super().__init__(message) + self.trace = trace + + +class OperatorInstantiationError(RuntimeError): + pass + + +class SemanticOperatorCatalog: + def __init__(self, descriptors: Iterable[OperatorBackendDescriptor] = ()): + self._descriptors: dict[tuple[str, str], OperatorBackendDescriptor] = {} + for descriptor in descriptors: + self.register_backend(descriptor) + + def register_backend( + self, + descriptor: OperatorBackendDescriptor, + *, + replace: bool = False, + ) -> None: + if not isinstance(descriptor, OperatorBackendDescriptor): + raise TypeError("descriptor must be an OperatorBackendDescriptor") + key = (descriptor.semantic_op, descriptor.backend_id) + if key in self._descriptors and not replace: + raise OperatorRegistrationError(f"operator backend is already registered: {key!r}") + self._descriptors[key] = descriptor + + def backend_descriptor( + self, + semantic_op: str, + backend_id: str, + ) -> Optional[OperatorBackendDescriptor]: + return self._descriptors.get((semantic_op.strip(), backend_id.strip())) + + def backend_descriptors( + self, + semantic_op: Optional[str] = None, + ) -> tuple[OperatorBackendDescriptor, ...]: + values: Iterable[OperatorBackendDescriptor] = self._descriptors.values() + if semantic_op is not None: + normalized = semantic_op.strip() + values = (value for value in values if value.semantic_op == normalized) + return tuple(sorted(values, key=lambda value: (value.semantic_op, value.backend_id))) + + def session(self, policy: _Policy = None) -> OperatorSession: + return OperatorSession(self, policy=policy) + + def _resolve( + self, + *, + semantic_op: str, + requested_backend: str, + target: str, + requirements: OperatorRequirements, + policy: OperatorResolutionPolicy, + ) -> OperatorResolution: + semantic_op = semantic_op.strip() + requested_backend = requested_backend.strip() + target = target.strip().lower() + if not semantic_op or not requested_backend or not target: + raise ValueError("semantic_op, requested_backend, and target must not be empty") + if not isinstance(requirements, OperatorRequirements): + raise TypeError("requirements must be an OperatorRequirements") + + descriptor = self.backend_descriptor(semantic_op, requested_backend) + if descriptor is None: + decision = _decision( + "registration", + requested_backend, + [item.backend_id for item in self.backend_descriptors(semantic_op)], + passed=False, + ) + trace = _trace( + semantic_op, + requested_backend, + target, + policy, + "unsupported", + (decision,), + ) + raise OperatorResolutionError( + f"exact operator backend {requested_backend!r} is not registered; " + "no fallback was attempted", + trace, + ) + + topology_ok = _supports( + descriptor.supported_topologies, + requirements.topology, + ) + decisions = ( + _decision("target", target, descriptor.supported_targets), + _decision( + "smoke_opt_in", + descriptor.is_smoke_only, + policy.allow_test_backends, + not descriptor.is_smoke_only or policy.allow_test_backends, + ), + _decision("device", requirements.device, descriptor.supported_devices), + _decision("dtype", requirements.dtype, descriptor.supported_dtypes), + _decision( + "topology", + requirements.topology, + descriptor.supported_topologies, + topology_ok, + ), + _decision( + "alignment_properties", + requirements.alignment_properties, + descriptor.determinism_or_alignment_properties, + ), + _decision( + "strict_observability", + policy.strict, + descriptor.is_strictly_observable, + not policy.strict or descriptor.is_strictly_observable, + ), + _decision( + "fallback_policy", + "error" if policy.strict else "declared", + descriptor.fallback_policy.value, + not policy.strict or descriptor.fallback_policy is OperatorFallbackPolicy.ERROR, + ), + ) + failed = tuple(item for item in decisions if not item.passed) + observable = descriptor.is_strictly_observable + status = "unsupported" if failed else ("resolved" if observable else "unobservable") + trace = _trace( + semantic_op, + requested_backend, + target, + policy, + status, + decisions, + descriptor, + ) + if failed: + raise OperatorResolutionError( + f"exact operator backend {requested_backend!r} is unsupported: " + + "; ".join(item.reason for item in failed), + trace, + ) + return OperatorResolution(descriptor, requirements, target, policy.strict, trace) + + +class OperatorSession: + def __init__(self, catalog: SemanticOperatorCatalog, policy: _Policy = None): + if not isinstance(catalog, SemanticOperatorCatalog): + raise TypeError("catalog must be a SemanticOperatorCatalog") + self.catalog = catalog + self.policy = policy or OperatorResolutionPolicy() + self._cache: dict[str, Any] = {} + self._records: dict[int, _InstanceRecord] = {} + + def resolve( + self, + *, + semantic_op: str, + requested_backend: str, + target: str, + requirements: OperatorRequirements, + policy: _Policy = None, + strict: Optional[bool] = None, + ) -> OperatorResolution: + return self.catalog._resolve( + semantic_op=semantic_op, + requested_backend=requested_backend, + target=target, + requirements=requirements, + policy=_resolve_policy(policy or self.policy, strict), + ) + + def instantiate( + self, + resolution: OperatorResolution, + *, + factory_kwargs: Optional[Mapping[str, Any]] = None, + cache: bool = False, + ) -> Any: + if not isinstance(resolution, OperatorResolution): + raise TypeError("resolution must be an OperatorResolution") + descriptor = resolution.descriptor + implementation = descriptor.implementation_class_or_factory + if resolution.trace.status != "resolved" or implementation is None: + raise OperatorInstantiationError( + f"backend {descriptor.backend_id!r} has no exact implementation" + ) + options = dict(factory_kwargs or {}) + cache_key = _fingerprint( + { + "descriptor": descriptor.descriptor_fingerprint, + "target": resolution.target, + "requirements": resolution.requirements.to_dict(), + "options": options, + } + ) + if cache and cache_key in self._cache: + return self._cache[cache_key] + factory = _load_factory(implementation) + try: + instance = factory(**options) + except Exception as exc: + raise OperatorInstantiationError( + f"failed to instantiate backend {descriptor.backend_id!r}: {exc}" + ) from exc + if instance is None: + raise OperatorInstantiationError("operator factory returned None") + self._records[id(instance)] = _InstanceRecord( + instance, + descriptor.descriptor_fingerprint, + resolution.target, + factory, + _freeze(options), + ) + if cache: + self._cache[cache_key] = instance + return instance + + def instance_provenance( + self, + resolution: OperatorResolution, + instance: Any, + ) -> OperatorInstanceProvenance: + descriptor = resolution.descriptor + record = self._records.get(id(instance)) + if ( + record is None + or record.instance is not instance + or record.descriptor_fingerprint != descriptor.descriptor_fingerprint + or record.target != resolution.target + ): + raise OperatorInstantiationError( + "operator instance does not match this session resolution" + ) + factory_reference = descriptor.implementation_reference + concrete = _reference(type(instance)) + if factory_reference is None or concrete is None: + raise OperatorInstantiationError("operator implementation is not observable") + options_fingerprint = _fingerprint(record.factory_options) + implementation_fingerprint = operator_implementation_fingerprint( + record.factory, + instance, + ) + instance_fingerprint = operator_instance_fingerprint( + descriptor_fingerprint=descriptor.descriptor_fingerprint, + factory_reference=factory_reference, + concrete_implementation=concrete, + implementation_fingerprint=implementation_fingerprint, + factory_options_fingerprint=options_fingerprint, + ) + return OperatorInstanceProvenance( + descriptor.semantic_op, + descriptor.backend_id, + resolution.target, + factory_reference, + concrete, + descriptor.descriptor_fingerprint, + implementation_fingerprint, + instance_fingerprint, + record.factory_options, + options_fingerprint, + ) + + def clear_instance_cache(self) -> None: + self._cache.clear() + + +def operator_implementation_fingerprint( + implementation: str | Callable[..., Any], + instance: Any, +) -> str: + return implementation_fingerprint( + implementation, + instance=instance, + entrypoints=("apply_fp32", "__call__"), + ) + + +def implementation_fingerprint( + implementation: str | Callable[..., Any], + *, + instance: Any = None, + entrypoints: Sequence[str] = (), +) -> str: + """Fingerprint executable code, not only its import reference. + + The identity includes source or bytecode for the resolved factory, its + concrete class, the defining modules, and explicitly named runtime entry + points. Module content covers helper functions called by an entry point; + callable identities additionally make in-process replacements observable. + """ + + factory = _load_factory(implementation) + concrete_type = type(instance) if instance is not None else None + runtime_entrypoints = {} + if instance is not None: + for name in sorted(set(entrypoints)): + value = getattr(instance, name, None) + if callable(value): + runtime_entrypoints[name] = _callable_identity(value) + return _fingerprint( + { + "factory": _implementation_identity(factory), + "concrete_type": ( + _implementation_identity(concrete_type) if concrete_type is not None else None + ), + "runtime_entrypoints": runtime_entrypoints, + } + ) + + +def operator_instance_fingerprint(**identity: str) -> str: + return _fingerprint(identity) + + +def _trace( + semantic_op: str, + backend: str, + target: str, + policy: OperatorResolutionPolicy, + status: str, + decisions: tuple[OperatorCapabilityDecision, ...], + descriptor: Optional[OperatorBackendDescriptor] = None, +) -> OperatorResolutionTrace: + observable = descriptor is not None and descriptor.is_strictly_observable + return OperatorResolutionTrace( + semantic_op, + backend, + target, + policy.strict, + status, + ( + descriptor.backend_id + if descriptor is not None and observable and status != "unsupported" + else None + ), + descriptor.implementation_reference if descriptor else None, + descriptor.descriptor_fingerprint if descriptor else None, + decisions, + ) + + +def _decision( + capability: str, + requested: Any, + supported: Any, + passed: Optional[bool] = None, +) -> OperatorCapabilityDecision: + passed = _supports(supported, requested) if passed is None else passed + actionable = { + "smoke_opt_in": "smoke backend use requires explicit opt-in", + "strict_observability": "runtime-native implementation is not exactly observable", + "fallback_policy": "strict resolution forbids declared or runtime fallback", + } + return OperatorCapabilityDecision( + capability, + requested, + supported, + passed, + ( + f"{capability} is supported" + if passed + else actionable.get(capability, f"{capability} is unsupported") + ), + ) + + +def _supports(supported: Any, requested: Any) -> bool: + if isinstance(supported, str) and supported in {"*", "any"}: + return True + if isinstance(supported, Mapping): + if not isinstance(requested, Mapping): + return False + wildcard = supported.get("*") + return all( + _supports(supported.get(key, wildcard), value) + for key, value in requested.items() + if key in supported or wildcard is not None + ) and all(key in supported or wildcard is not None for key in requested) + if isinstance(supported, (set, frozenset, tuple, list)): + if isinstance(requested, (set, frozenset, tuple, list)): + return all(any(_supports(item, value) for item in supported) for value in requested) + return any(_supports(item, requested) for item in supported) + return supported == requested + + +def _resolve_policy(policy: _Policy, strict: Optional[bool]) -> OperatorResolutionPolicy: + policy = policy or OperatorResolutionPolicy() + return policy if strict is None else replace(policy, strict=strict) + + +def _load_factory(value: str | Callable[..., Any]) -> Callable[..., Any]: + if callable(value): + return value + try: + module_name, attribute = value.rsplit(".", 1) + factory = getattr(importlib.import_module(module_name), attribute) + except (ValueError, ImportError, AttributeError, ModuleNotFoundError) as exc: + raise OperatorInstantiationError(f"operator factory {value!r} is unavailable") from exc + if not callable(factory): + raise OperatorInstantiationError(f"operator factory {value!r} is not callable") + return factory + + +def _reference(value: Any) -> Optional[str]: + if value is None or isinstance(value, str): + return value + module = getattr(value, "__module__", type(value).__module__) + qualname = getattr(value, "__qualname__", type(value).__qualname__) + return f"{module}.{qualname}" + + +def _normalize_device(value: Any) -> str: + value = str(value).strip().lower() + if value.startswith("torch.device("): + value = value.removeprefix("torch.device(").removesuffix(")").strip("'\"") + if value.startswith("cuda:"): + return "cuda" + return {"gpu": "cuda", "hip": "rocm"}.get(value, value) + + +def _normalize_dtype(value: Any) -> str: + value = str(value).strip().lower().replace("torch.", "") + return { + "fp32": "float32", + "float": "float32", + "bf16": "bfloat16", + "fp16": "float16", + "half": "float16", + }.get(value, value) + + +def _normalized_values(values: Iterable[Any], normalize: Callable[[Any], str]) -> frozenset[str]: + return frozenset(value for item in values if (value := normalize(item).strip().lower())) + + +def _freeze(value: Any) -> Any: + if isinstance(value, Mapping): + return MappingProxyType({str(key): _freeze(item) for key, item in value.items()}) + if isinstance(value, (tuple, list)): + return tuple(_freeze(item) for item in value) + if isinstance(value, (set, frozenset)): + return frozenset(_freeze(item) for item in value) + return value + + +def _json_value(value: Any) -> Any: + if isinstance(value, _JsonRecord): + return value.to_dict() + if isinstance(value, Enum): + return value.value + if isinstance(value, Mapping): + return {str(key): _json_value(item) for key, item in sorted(value.items())} + if isinstance(value, (set, frozenset)): + return sorted((_json_value(item) for item in value), key=repr) + if isinstance(value, (tuple, list)): + return [_json_value(item) for item in value] + if callable(value): + return _reference(value) + return value + + +def _implementation_identity(value: Any) -> Mapping[str, Any]: + reference = _reference(value) + identity: dict[str, Any] = { + "reference": reference, + "kind": "class" if inspect.isclass(value) else "callable", + "callable": _callable_identity(value), + "module": _module_identity(getattr(value, "__module__", None)), + } + if inspect.isclass(value): + identity["members"] = { + name: _callable_identity(member) + for name, raw_member in sorted(vars(value).items()) + if (member := _descriptor_callable(raw_member)) is not None + } + return identity + + +def _descriptor_callable(value: Any) -> Optional[Callable[..., Any]]: + if isinstance(value, (classmethod, staticmethod)): + value = value.__func__ + elif isinstance(value, property): + return None + return value if callable(value) else None + + +def _callable_identity(value: Any) -> Mapping[str, Any]: + if inspect.ismethod(value): + value = value.__func__ + try: + unwrapped = inspect.unwrap(value) + except (TypeError, ValueError): + unwrapped = value + code = getattr(unwrapped, "__code__", None) + try: + source = inspect.getsource(unwrapped) + except (OSError, TypeError): + source = None + identity: dict[str, Any] = { + "reference": _reference(unwrapped), + "source_sha256": ( + hashlib.sha256(source.encode("utf-8")).hexdigest() if source is not None else None + ), + "code_sha256": _code_fingerprint(code) if isinstance(code, CodeType) else None, + } + if isinstance(code, CodeType): + identity["defaults"] = _code_value(getattr(unwrapped, "__defaults__", None)) + identity["keyword_defaults"] = _code_value(getattr(unwrapped, "__kwdefaults__", None)) + return identity + + +def _code_fingerprint(code: CodeType) -> str: + return _fingerprint( + { + "bytecode": code.co_code.hex(), + "constants": tuple(_code_value(value) for value in code.co_consts), + "names": code.co_names, + "variables": code.co_varnames, + "free_variables": code.co_freevars, + "cell_variables": code.co_cellvars, + "positional_arguments": code.co_argcount, + "positional_only_arguments": code.co_posonlyargcount, + "keyword_only_arguments": code.co_kwonlyargcount, + "flags": code.co_flags, + } + ) + + +def _code_value(value: Any) -> Any: + if isinstance(value, CodeType): + return {"nested_code_sha256": _code_fingerprint(value)} + if isinstance(value, bytes): + return {"bytes_sha256": hashlib.sha256(value).hexdigest()} + if isinstance(value, Mapping): + return { + str(key): _code_value(item) + for key, item in sorted(value.items(), key=lambda pair: repr(pair[0])) + } + if isinstance(value, (tuple, list)): + return [_code_value(item) for item in value] + if isinstance(value, (set, frozenset)): + return sorted((_code_value(item) for item in value), key=repr) + if value is None or isinstance(value, (bool, int, float, str)): + return value + return {"type": _reference(type(value)), "repr": repr(value)} + + +def _module_identity(module_name: Optional[str]) -> Optional[Mapping[str, Any]]: + if not module_name: + return None + try: + module = importlib.import_module(module_name) + except (ImportError, ModuleNotFoundError): + return {"name": module_name, "content_sha256": None} + module_file = getattr(module, "__file__", None) + if not module_file: + return {"name": module_name, "content_sha256": None} + path = Path(module_file) + try: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + except OSError: + return {"name": module_name, "content_sha256": None} + return { + "name": module_name, + "content_sha256": digest.hexdigest(), + } + + +def _fingerprint(value: Any) -> str: + encoded = json.dumps(_json_value(value), sort_keys=True, separators=(",", ":")) + return hashlib.sha256(encoded.encode("utf-8")).hexdigest() From 552270fd9edb17f8b9ce6bb7f8f5b5c81251ded6 Mon Sep 17 00:00:00 2001 From: CyberSecurityErial <2710555967@qq.com> Date: Sun, 19 Jul 2026 08:30:48 +0800 Subject: [PATCH 2/8] feat(alignment): define cross-configuration contracts and plans --- rl_engine/alignment/cross_config/_json.py | 44 ++ .../alignment/cross_config/comparison.py | 316 ++++++++++ rl_engine/alignment/cross_config/config.py | 424 +++++++++++++ .../alignment/cross_config/execution_plan.py | 77 +++ rl_engine/alignment/cross_config/planner.py | 543 ++++++++++++++++ rl_engine/alignment/cross_config/schema.py | 582 ++++++++++++++++++ rl_engine/kernels/gtest/tolerance.py | 57 +- 7 files changed, 2042 insertions(+), 1 deletion(-) create mode 100644 rl_engine/alignment/cross_config/_json.py create mode 100644 rl_engine/alignment/cross_config/comparison.py create mode 100644 rl_engine/alignment/cross_config/config.py create mode 100644 rl_engine/alignment/cross_config/execution_plan.py create mode 100644 rl_engine/alignment/cross_config/planner.py create mode 100644 rl_engine/alignment/cross_config/schema.py diff --git a/rl_engine/alignment/cross_config/_json.py b/rl_engine/alignment/cross_config/_json.py new file mode 100644 index 00000000..fe8eb876 --- /dev/null +++ b/rl_engine/alignment/cross_config/_json.py @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Fail-closed JSON decoding shared by configs and artifacts.""" + +from __future__ import annotations + +import json +import math +from typing import Any + + +def strict_json_loads(value: str) -> Any: + """Decode RFC JSON while rejecting duplicate keys and non-finite numbers.""" + + return json.loads( + value, + object_pairs_hook=_unique_object, + parse_constant=_reject_json_constant, + parse_float=_parse_finite_float, + ) + + +def _unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON key {key!r}") + result[key] = value + return result + + +def _reject_json_constant(value: str) -> Any: + raise ValueError(f"non-finite JSON constant {value!r} is forbidden") + + +def _parse_finite_float(value: str) -> float: + result = float(value) + if not math.isfinite(result): + raise ValueError(f"non-finite JSON number {value!r} is forbidden") + return result + + +__all__ = ["strict_json_loads"] diff --git a/rl_engine/alignment/cross_config/comparison.py b/rl_engine/alignment/cross_config/comparison.py new file mode 100644 index 00000000..c6db2d62 --- /dev/null +++ b/rl_engine/alignment/cross_config/comparison.py @@ -0,0 +1,316 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Fixed-contract selected-token comparison for cross-configuration cases.""" + +from __future__ import annotations + +import math +from dataclasses import fields +from typing import Any + +import torch + +from rl_engine.alignment.cross_config.schema import ( + AlignmentResult, + AlignmentStatus, + ScoreArtifact, + ScoreSide, + SemanticIdentitySpec, + TokenComparisonArtifact, +) +from rl_engine.kernels.gtest.tolerance import ( + resolve_logprob_threshold, + tolerance_contract_fingerprint, +) + + +def semantic_identity_errors( + rollout: SemanticIdentitySpec, + training: SemanticIdentitySpec, +) -> tuple[str, ...]: + """Return every logical identity field that differs between the two sides.""" + + return tuple( + item.name + for item in fields(SemanticIdentitySpec) + if getattr(rollout, item.name) != getattr(training, item.name) + ) + + +def recompute_mismatch_mask( + rollout_logprobs: torch.Tensor, + training_logprobs: torch.Tensor, + active_mask: torch.Tensor, + fixed_threshold: float, +) -> torch.Tensor: + """Recompute the sole token mismatch signal from persisted tensors.""" + + if rollout_logprobs.shape != training_logprobs.shape: + raise ValueError("rollout and training logprobs must have identical shapes") + if active_mask.shape != rollout_logprobs.shape: + raise ValueError("active_mask shape must match selected logprobs") + if fixed_threshold < 0.0: + raise ValueError("fixed_threshold must be non-negative") + active = active_mask.to(device=rollout_logprobs.device, dtype=torch.bool) + training = training_logprobs.to(device=rollout_logprobs.device) + return active & (torch.abs(training - rollout_logprobs) > fixed_threshold) + + +class FixedThresholdComparator: + """Compare paired selected logprobs using only the current WS1 contract.""" + + def compare(self, rollout: ScoreArtifact, training: ScoreArtifact) -> AlignmentResult: + contract_fingerprint = tolerance_contract_fingerprint() + artifact_errors = _artifact_errors(rollout, training) + if artifact_errors: + return AlignmentResult( + case_id=rollout.case_id, + attempt_id=rollout.attempt_id, + status=AlignmentStatus.INVALID_ARTIFACT, + comparable=False, + passed=False, + active_token_count=0, + mismatch_count=0, + contract_fingerprint=contract_fingerprint, + artifact_errors=artifact_errors, + ) + + identity_errors = list(semantic_identity_errors(rollout.identity, training.identity)) + identity_errors.extend(_artifact_identity_errors(rollout, training)) + if identity_errors: + return AlignmentResult( + case_id=rollout.case_id, + attempt_id=rollout.attempt_id, + status=AlignmentStatus.INVALID_IDENTITY, + comparable=False, + passed=False, + active_token_count=0, + mismatch_count=0, + contract_fingerprint=contract_fingerprint, + identity_errors=tuple(dict.fromkeys(identity_errors)), + ) + + threshold, threshold_error = _resolve_fixed_threshold(rollout, training) + if threshold_error is not None: + return AlignmentResult( + case_id=rollout.case_id, + attempt_id=rollout.attempt_id, + status=AlignmentStatus.INVALID_ARTIFACT, + comparable=False, + passed=False, + active_token_count=0, + mismatch_count=0, + contract_fingerprint=contract_fingerprint, + artifact_errors=(threshold_error,), + ) + assert threshold is not None + fixed_threshold = threshold + + rollout_logprobs = rollout.selected_logprobs.detach().cpu() + training_logprobs = training.selected_logprobs.detach().cpu() + active_mask = rollout.active_mask.detach().cpu().to(dtype=torch.bool) + active_token_count = int(active_mask.sum().item()) + if active_token_count: + active_rollout = rollout_logprobs[active_mask] + active_training = training_logprobs[active_mask] + if not bool(torch.isfinite(active_rollout).all().item()) or not bool( + torch.isfinite(active_training).all().item() + ): + return AlignmentResult( + case_id=rollout.case_id, + attempt_id=rollout.attempt_id, + status=AlignmentStatus.INVALID_ARTIFACT, + comparable=False, + passed=False, + active_token_count=active_token_count, + mismatch_count=0, + contract_fingerprint=contract_fingerprint, + fixed_threshold=fixed_threshold, + artifact_errors=("active selected logprobs must be finite",), + ) + + absolute_diff = torch.abs(training_logprobs - rollout_logprobs) + mismatch_mask = recompute_mismatch_mask( + rollout_logprobs, + training_logprobs, + active_mask, + fixed_threshold, + ) + token_artifact = TokenComparisonArtifact( + rollout_logprobs=rollout_logprobs, + training_logprobs=training_logprobs, + active_mask=active_mask, + absolute_diff=absolute_diff, + mismatch_mask=mismatch_mask, + fixed_threshold=fixed_threshold, + ) + if active_token_count == 0: + return AlignmentResult( + case_id=rollout.case_id, + attempt_id=rollout.attempt_id, + status=AlignmentStatus.ZERO_ACTIVE_TOKENS, + comparable=False, + passed=False, + active_token_count=0, + mismatch_count=0, + contract_fingerprint=contract_fingerprint, + fixed_threshold=fixed_threshold, + token_artifact=token_artifact, + ) + + mismatch_count = int(mismatch_mask.sum().item()) + passed = mismatch_count == 0 + return AlignmentResult( + case_id=rollout.case_id, + attempt_id=rollout.attempt_id, + status=AlignmentStatus.PASS if passed else AlignmentStatus.FAIL, + comparable=True, + passed=passed, + active_token_count=active_token_count, + mismatch_count=mismatch_count, + contract_fingerprint=contract_fingerprint, + fixed_threshold=fixed_threshold, + diagnostics=_diagnostics( + rollout_logprobs, + training_logprobs, + active_mask, + absolute_diff, + mismatch_count, + ), + token_artifact=token_artifact, + ) + + +def compare_score_artifacts( + rollout: ScoreArtifact, + training: ScoreArtifact, +) -> AlignmentResult: + """Convenience wrapper whose API deliberately exposes no threshold override.""" + + return FixedThresholdComparator().compare(rollout, training) + + +def _resolve_fixed_threshold( + rollout: ScoreArtifact, + training: ScoreArtifact, +) -> tuple[float | None, str | None]: + """Resolve one WS1 threshold, rejecting any mixed-dtype ambiguity.""" + + try: + rollout_threshold = resolve_logprob_threshold(rollout.scorer.dtype) + training_threshold = resolve_logprob_threshold(training.scorer.dtype) + except ValueError as exc: + return None, f"fixed WS1 threshold is unavailable: {exc}" + if rollout_threshold != training_threshold: + return ( + None, + "fixed WS1 threshold is ambiguous for scorer dtypes " + f"rollout={rollout.scorer.dtype!r}, training={training.scorer.dtype!r}", + ) + return rollout_threshold, None + + +def _artifact_errors(rollout: ScoreArtifact, training: ScoreArtifact) -> tuple[str, ...]: + errors: list[str] = [] + if rollout.side is not ScoreSide.ROLLOUT: + errors.append("first artifact side must be rollout") + if training.side is not ScoreSide.TRAINING: + errors.append("second artifact side must be training") + if rollout.case_id != training.case_id: + errors.append("case_id") + if rollout.attempt_id != training.attempt_id: + errors.append("attempt_id") + if rollout.selected_logprobs.shape != training.selected_logprobs.shape: + errors.append("selected_logprobs shape") + for label, artifact in (("rollout", rollout), ("training", training)): + expected_dtype = _score_dtype(artifact.scorer.dtype) + if not artifact.selected_logprobs.is_floating_point(): + errors.append(f"{label}.selected_logprobs must be floating point") + elif expected_dtype is None: + errors.append(f"{label}.scorer dtype is unsupported") + elif artifact.selected_logprobs.dtype != expected_dtype: + errors.append( + f"{label}.selected_logprobs dtype does not match scorer dtype " + f"({artifact.selected_logprobs.dtype} != {expected_dtype})" + ) + return tuple(errors) + + +def _score_dtype(value: str) -> torch.dtype | None: + normalized = str(value).strip().lower().removeprefix("torch.") + return { + "float32": torch.float32, + "fp32": torch.float32, + "float16": torch.float16, + "fp16": torch.float16, + "bfloat16": torch.bfloat16, + "bf16": torch.bfloat16, + "float64": torch.float64, + }.get(normalized) + + +def _artifact_identity_errors( + rollout: ScoreArtifact, + training: ScoreArtifact, +) -> tuple[str, ...]: + errors: list[str] = [] + rollout_identity_mask = _identity_mask(rollout.identity) + training_identity_mask = _identity_mask(training.identity) + rollout_mask = rollout.active_mask.detach().cpu().to(dtype=torch.bool) + training_mask = training.active_mask.detach().cpu().to(dtype=torch.bool) + if rollout_mask.shape != rollout_identity_mask.shape or not torch.equal( + rollout_mask, rollout_identity_mask + ): + errors.append("rollout.active_mask") + if training_mask.shape != training_identity_mask.shape or not torch.equal( + training_mask, training_identity_mask + ): + errors.append("training.active_mask") + if rollout_mask.shape != training_mask.shape or not torch.equal(rollout_mask, training_mask): + errors.append("active_mask") + return tuple(errors) + + +def _identity_mask(identity: SemanticIdentitySpec) -> torch.Tensor: + return torch.tensor(identity.active_mask, dtype=torch.bool) + + +def _diagnostics( + rollout_logprobs: torch.Tensor, + training_logprobs: torch.Tensor, + active_mask: torch.Tensor, + absolute_diff: torch.Tensor, + mismatch_count: int, +) -> dict[str, Any]: + active_diff = absolute_diff[active_mask].float() + delta = (training_logprobs[active_mask] - rollout_logprobs[active_mask]).float() + worst_active_index = int(torch.argmax(active_diff).item()) + active_coordinates = torch.nonzero(active_mask, as_tuple=False) + worst_coordinate = tuple(int(item) for item in active_coordinates[worst_active_index].tolist()) + approximate_kl = torch.exp(delta.double()) - delta.double() - 1.0 + approximate_kl_mean = _finite_float_or_none(approximate_kl.mean()) + active_count = int(active_diff.numel()) + return { + "mean_abs_diff": _finite_float_or_none(active_diff.mean()), + "p95_abs_diff": _finite_float_or_none(torch.quantile(active_diff, 0.95)), + "p99_abs_diff": _finite_float_or_none(torch.quantile(active_diff, 0.99)), + "max_abs_diff": _finite_float_or_none(active_diff.max()), + "mismatch_ratio": mismatch_count / active_count, + "approximate_kl_mean": approximate_kl_mean, + "approximate_kl_finite": approximate_kl_mean is not None, + "worst_token_index": worst_coordinate, + } + + +def _finite_float_or_none(value: torch.Tensor) -> float | None: + result = float(value.item()) + return result if math.isfinite(result) else None + + +__all__ = [ + "FixedThresholdComparator", + "compare_score_artifacts", + "recompute_mismatch_mask", + "semantic_identity_errors", +] diff --git a/rl_engine/alignment/cross_config/config.py b/rl_engine/alignment/cross_config/config.py new file mode 100644 index 00000000..e02c2198 --- /dev/null +++ b/rl_engine/alignment/cross_config/config.py @@ -0,0 +1,424 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Strict, dependency-free experiment configuration.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, field +from dataclasses import fields as dataclass_fields +from pathlib import Path +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, Mapping + +from rl_engine.alignment.cross_config._json import strict_json_loads +from rl_engine.alignment.cross_config.planner import normalize_backend_id +from rl_engine.alignment.cross_config.schema import ( + ExperimentCase, + ExperimentDefinition, + InterventionSpec, + PlanningStrategy, + SemanticIdentitySpec, +) + +if TYPE_CHECKING: + from rl_engine.alignment.cross_config.planner import ExperimentPlan + + +CONFIG_SCHEMA_VERSION = "cross_config.experiment_config.v1" +_FORBIDDEN_THRESHOLD_KEYS = frozenset({"threshold", "fixed_threshold", "tolerance", "atol", "rtol"}) +_TOP_LEVEL_KEYS = frozenset( + { + "schema_version", + "experiment_id", + "scenario_id", + "contract_source", + "contract_version", + "strategy", + "strict_fallback", + "identity", + "baseline", + "interventions", + "pairwise_paths", + "operators", + "scenario", + } +) +_IDENTITY_KEYS = frozenset( + item.name for item in dataclass_fields(SemanticIdentitySpec) if item.name != "schema_version" +) +_INTERVENTION_KEYS = frozenset({"path", "values"}) +_OPERATOR_NAMES = frozenset({"selected_logprob"}) +_OPERATOR_TARGETS = frozenset({"rollout", "training"}) +_OPERATOR_BINDING_KEYS = frozenset({"backend", "options"}) + + +@dataclass(frozen=True) +class OperatorSelection: + """Concrete selected-logprob implementation requested for each scorer side. + + ``logp.backend`` remains the concise both-sides shortcut. This explicit form + is needed only when rollout and training intentionally use different + implementations. + """ + + rollout_backend: str + training_backend: str + rollout_options: Mapping[str, Any] = field(default_factory=dict) + training_options: Mapping[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + object.__setattr__(self, "rollout_backend", normalize_backend_id(self.rollout_backend)) + object.__setattr__( + self, + "training_backend", + normalize_backend_id(self.training_backend), + ) + object.__setattr__(self, "rollout_options", _freeze_mapping(self.rollout_options)) + object.__setattr__(self, "training_options", _freeze_mapping(self.training_options)) + + def backend_for(self, target: str) -> str: + if target == "rollout": + return self.rollout_backend + if target == "training": + return self.training_backend + raise ValueError("operator target must be 'rollout' or 'training'") + + def options_for(self, target: str) -> Mapping[str, Any]: + if target == "rollout": + return self.rollout_options + if target == "training": + return self.training_options + raise ValueError("operator target must be 'rollout' or 'training'") + + def to_dict(self) -> dict[str, Any]: + return { + "selected_logprob": { + "rollout": { + "backend": self.rollout_backend, + "options": _plain_value(self.rollout_options), + }, + "training": { + "backend": self.training_backend, + "options": _plain_value(self.training_options), + }, + } + } + + +@dataclass(frozen=True) +class ExperimentConfig: + """Loaded experiment plus optional target-specific operator selection.""" + + definition: ExperimentDefinition + source_path: Path + operators: OperatorSelection | None = None + schema_version: str = CONFIG_SCHEMA_VERSION + + def to_dict(self) -> dict[str, Any]: + """Return the normalized, portable experiment-config representation.""" + + payload = self.definition.to_dict() + payload["schema_version"] = self.schema_version + if self.operators is not None: + payload["operators"] = self.operators.to_dict() + return payload + + def plan(self) -> ExperimentPlan: + """Build the deterministic plan without importing a runtime backend.""" + + from rl_engine.alignment.cross_config.planner import Planner + + return Planner().plan(self.definition) + + def operators_for(self, case: ExperimentCase) -> OperatorSelection: + """Resolve the concise ``logp.backend`` shortcut for one planned case.""" + + backend = _case_logp_backend(case) + if self.operators is None: + return OperatorSelection(backend, backend) + if self.operators.rollout_backend != backend: + raise ValueError( + "operators.selected_logprob.rollout must match the planned " + f"logp.backend: {self.operators.rollout_backend!r} != {backend!r}" + ) + return self.operators + + +def load_config(path: str | Path) -> ExperimentConfig: + """Load one versioned JSON experiment with no threshold override surface.""" + + source = Path(path) + try: + raw = strict_json_loads(source.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise ValueError(f"failed to load cross-configuration config {source}: {exc}") from exc + if not isinstance(raw, dict): + raise ValueError("cross-configuration config must contain a JSON object") + _reject_unknown_keys(raw, _TOP_LEVEL_KEYS, "config") + _reject_threshold_keys(raw) + if raw.get("schema_version") != CONFIG_SCHEMA_VERSION: + raise ValueError( + f"unsupported cross-configuration config schema {raw.get('schema_version')!r}; " + f"expected {CONFIG_SCHEMA_VERSION!r}" + ) + + identity_raw = _required_mapping(raw, "identity") + _reject_unknown_keys(identity_raw, _IDENTITY_KEYS, "identity") + scenario = _optional_mapping(raw, "scenario") + _reject_scenario_controls(scenario) + + interventions_raw = raw.get("interventions", []) + if not isinstance(interventions_raw, list): + raise ValueError("interventions must be a list") + interventions = tuple(_load_intervention(item) for item in interventions_raw) + + pairwise_raw = raw.get("pairwise_paths", []) + if not isinstance(pairwise_raw, list): + raise ValueError("pairwise_paths must be a list") + pairwise_paths = tuple(_load_pair(item) for item in pairwise_raw) + + strict_fallback = raw.get("strict_fallback", True) + if not isinstance(strict_fallback, bool): + raise ValueError("strict_fallback must be a JSON boolean") + + definition = ExperimentDefinition( + experiment_id=_required_string(raw, "experiment_id"), + scenario_id=_required_string(raw, "scenario_id"), + identity=SemanticIdentitySpec(**identity_raw), + baseline=_required_mapping(raw, "baseline"), + interventions=interventions, + scenario=scenario, + strategy=PlanningStrategy(raw.get("strategy", "one_at_a_time")), + strict_fallback=strict_fallback, + pairwise_paths=pairwise_paths, + contract_source=raw.get("contract_source", "ws1"), + contract_version=raw.get("contract_version", "current"), + ) + operators = _load_operators(raw.get("operators")) + if operators is not None: + if any(item.path == "logp.backend" for item in interventions): + raise ValueError( + "explicit operators cannot be combined with logp.backend interventions; " + "use the shortcut or one fixed target mapping" + ) + baseline_backend = _definition_logp_backend(definition) + if operators.rollout_backend != baseline_backend: + raise ValueError( + "operators.selected_logprob.rollout must match baseline logp.backend: " + f"{operators.rollout_backend!r} != {baseline_backend!r}" + ) + + return ExperimentConfig( + definition=definition, + operators=operators, + source_path=source, + ) + + +def bind_operator_selection( + case: ExperimentCase, + selection: OperatorSelection, +) -> ExperimentCase: + """Bind target-specific operators into the execution identity. + + Planning remains semantic-operator agnostic; the immutable binding extends + the case and resume key before any runtime is created. + """ + + requested_backend = _case_logp_backend(case) + if selection.rollout_backend != requested_backend: + raise ValueError( + "rollout operator must match the planned logp.backend: " + f"{selection.rollout_backend!r} != {requested_backend!r}" + ) + binding = selection.to_dict() + payload = { + "base_case_id": case.case_id, + "base_scenario_fingerprint": case.scenario_fingerprint, + "operators": binding, + } + serialized = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + operator_fingerprint = hashlib.sha256(serialized).hexdigest() + case_hash = hashlib.sha256( + json.dumps( + {"base_case_id": case.case_id, "operator_fingerprint": operator_fingerprint}, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + ).hexdigest()[:24] + scenario_fingerprint = hashlib.sha256( + f"{case.scenario_fingerprint}:{operator_fingerprint}".encode("utf-8") + ).hexdigest() + return ExperimentCase( + case_id=f"cross-config-{case_hash}", + experiment_id=case.experiment_id, + scenario_id=case.scenario_id, + identity=case.identity, + requested=case.requested, + execution_binding={"operators": binding}, + changed_paths=case.changed_paths, + contract_fingerprint=case.contract_fingerprint, + scenario_fingerprint=scenario_fingerprint, + ) + + +def _load_intervention(value: Any) -> InterventionSpec: + if not isinstance(value, Mapping): + raise ValueError("each intervention must be an object") + _reject_unknown_keys(value, _INTERVENTION_KEYS, "intervention") + values = value.get("values") + if not isinstance(values, list): + raise ValueError("intervention values must be a list") + return InterventionSpec(path=_required_string(value, "path"), values=tuple(values)) + + +def _load_pair(value: Any) -> tuple[str, str]: + if ( + not isinstance(value, list) + or len(value) != 2 + or not all(isinstance(item, str) for item in value) + ): + raise ValueError("each pairwise_paths entry must contain exactly two string paths") + return value[0], value[1] + + +def _load_operators(value: Any) -> OperatorSelection | None: + if value is None: + return None + if not isinstance(value, Mapping): + raise ValueError("operators must be an object") + _reject_unknown_keys(value, _OPERATOR_NAMES, "operators") + selected = value.get("selected_logprob") + if not isinstance(selected, Mapping): + raise ValueError("operators.selected_logprob must be an object") + _reject_unknown_keys(selected, _OPERATOR_TARGETS, "operators.selected_logprob") + rollout_backend, rollout_options = _load_operator_binding(selected, "rollout") + training_backend, training_options = _load_operator_binding(selected, "training") + return OperatorSelection( + rollout_backend=rollout_backend, + training_backend=training_backend, + rollout_options=rollout_options, + training_options=training_options, + ) + + +def _load_operator_binding( + value: Mapping[str, Any], + target: str, +) -> tuple[str, Mapping[str, Any]]: + binding = value.get(target) + if isinstance(binding, str): + if not binding.strip(): + raise ValueError(f"operators.selected_logprob.{target} must not be empty") + return binding, {} + if not isinstance(binding, Mapping): + raise ValueError(f"operators.selected_logprob.{target} must be a backend string or object") + _reject_unknown_keys(binding, _OPERATOR_BINDING_KEYS, f"{target} operator binding") + return _required_string(binding, "backend"), _optional_mapping(binding, "options") + + +def _case_logp_backend(case: ExperimentCase) -> str: + logp = case.requested.get("logp") + backend = logp.get("backend") if isinstance(logp, Mapping) else None + if not isinstance(backend, str) or not backend: + raise ValueError("planned cases must contain a non-empty string logp.backend") + return normalize_backend_id(backend) + + +def _definition_logp_backend(definition: ExperimentDefinition) -> str: + logp = definition.baseline.get("logp") + backend = logp.get("backend") if isinstance(logp, Mapping) else None + if not isinstance(backend, str) or not backend: + raise ValueError("baseline must contain a non-empty string logp.backend") + return normalize_backend_id(backend) + + +def _reject_scenario_controls(scenario: Mapping[str, Any]) -> None: + behavior_keys = sorted( + set(scenario).intersection( + {"execution", "plan_only", "operator_cases", "expected_status", "allow_smoke_operators"} + ) + ) + if behavior_keys: + raise ValueError( + "scenario is metadata only; move execution and operator policy to the CLI/config: " + f"{behavior_keys}" + ) + + +def _reject_threshold_keys(value: Any, prefix: str = "") -> None: + if isinstance(value, Mapping): + for key, child in value.items(): + normalized = str(key).strip().lower() + path = f"{prefix}.{key}" if prefix else str(key) + if normalized in _FORBIDDEN_THRESHOLD_KEYS: + raise ValueError( + f"{path} is forbidden: the fixed numerical-contract threshold is imported" + ) + _reject_threshold_keys(child, path) + elif isinstance(value, list): + for index, child in enumerate(value): + _reject_threshold_keys(child, f"{prefix}[{index}]") + + +def _reject_unknown_keys( + value: Mapping[str, Any], + allowed: frozenset[str], + label: str, +) -> None: + unknown = sorted(set(value).difference(allowed)) + if unknown: + raise ValueError(f"unknown {label} keys: {unknown}") + + +def _required_mapping(value: Mapping[str, Any], key: str) -> dict[str, Any]: + child = value.get(key) + if not isinstance(child, Mapping): + raise ValueError(f"{key} must be an object") + return dict(child) + + +def _optional_mapping(value: Mapping[str, Any], key: str) -> dict[str, Any]: + child = value.get(key, {}) + if not isinstance(child, Mapping): + raise ValueError(f"{key} must be an object") + return dict(child) + + +def _required_string(value: Mapping[str, Any], key: str) -> str: + child = value.get(key) + if not isinstance(child, str) or not child.strip(): + raise ValueError(f"{key} must be a non-empty string") + return child.strip() + + +def _freeze_mapping(value: Mapping[str, Any]) -> Mapping[str, Any]: + return MappingProxyType({str(key): _freeze_value(child) for key, child in value.items()}) + + +def _freeze_value(value: Any) -> Any: + if isinstance(value, Mapping): + return _freeze_mapping(value) + if isinstance(value, list): + return tuple(_freeze_value(child) for child in value) + return value + + +def _plain_value(value: Any) -> Any: + if isinstance(value, Mapping): + return {str(key): _plain_value(child) for key, child in value.items()} + if isinstance(value, (tuple, list)): + return [_plain_value(child) for child in value] + return value + + +__all__ = [ + "CONFIG_SCHEMA_VERSION", + "ExperimentConfig", + "OperatorSelection", + "bind_operator_selection", + "load_config", +] diff --git a/rl_engine/alignment/cross_config/execution_plan.py b/rl_engine/alignment/cross_config/execution_plan.py new file mode 100644 index 00000000..d3be05b2 --- /dev/null +++ b/rl_engine/alignment/cross_config/execution_plan.py @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Canonical, runtime-independent execution-plan construction.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Mapping + +from rl_engine.alignment.cross_config.config import ( + ExperimentConfig, + OperatorSelection, + bind_operator_selection, +) +from rl_engine.alignment.cross_config.planner import PlanningIssue +from rl_engine.alignment.cross_config.schema import ExperimentCase + + +@dataclass(frozen=True) +class ExecutionPlanEntry: + """One operator-bound case and its resolved operator selection.""" + + case: ExperimentCase + operators: OperatorSelection + schema_version: str = "cross_config.execution_plan_entry.v1" + + def to_dict(self) -> dict[str, Any]: + """Return the canonical append-only plan row.""" + + return { + "schema_version": self.schema_version, + "case": self.case.to_dict(), + "operators": self.operators.to_dict(), + } + + +@dataclass(frozen=True) +class ExecutionPlan: + """Canonical metadata shared by planning and every runtime adapter.""" + + experiment: Mapping[str, Any] + entries: tuple[ExecutionPlanEntry, ...] + issues: tuple[PlanningIssue, ...] = () + schema_version: str = "cross_config.execution_plan.v1" + + def rows(self) -> tuple[dict[str, Any], ...]: + """Serialize all plan entries in deterministic execution order.""" + + return tuple(entry.to_dict() for entry in self.entries) + + +def build_execution_plan(config: ExperimentConfig) -> ExecutionPlan: + """Plan, resolve operators, and bind them into immutable case identities.""" + + planned = config.plan() + entries: list[ExecutionPlanEntry] = [] + for case in planned.cases: + operators = config.operators_for(case) + entries.append( + ExecutionPlanEntry( + case=bind_operator_selection(case, operators), + operators=operators, + ) + ) + return ExecutionPlan( + experiment=config.to_dict(), + entries=tuple(entries), + issues=planned.issues, + ) + + +__all__ = [ + "ExecutionPlan", + "ExecutionPlanEntry", + "build_execution_plan", +] diff --git a/rl_engine/alignment/cross_config/planner.py b/rl_engine/alignment/cross_config/planner.py new file mode 100644 index 00000000..2b7ba8f8 --- /dev/null +++ b/rl_engine/alignment/cross_config/planner.py @@ -0,0 +1,543 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Typed baseline, one-at-a-time, and explicitly bounded pairwise planning.""" + +from __future__ import annotations + +import hashlib +import itertools +import json +from dataclasses import dataclass +from typing import Any, Callable, Mapping, Optional, Sequence + +from rl_engine.alignment.cross_config.schema import ( + ExperimentCase, + ExperimentDefinition, + IsolationScope, + KnobDescriptor, + PlanningStrategy, +) +from rl_engine.kernels.gtest.tolerance import tolerance_contract_fingerprint + +Normalizer = Callable[[Any], Any] +Constraint = Callable[[str, Any, Mapping[str, Any]], Optional["PlanningIssue"]] + + +@dataclass(frozen=True) +class PlanningIssue: + """Structured planning rejection that callers can persist or display.""" + + code: str + reason: str + path: Optional[str] = None + value: Any = None + + def to_dict(self) -> dict[str, Any]: + return { + "code": self.code, + "reason": self.reason, + "path": self.path, + "value": self.value, + } + + +class PlanningError(ValueError): + """Raised for an invalid experiment definition with structured issues.""" + + def __init__(self, issues: Sequence[PlanningIssue]): + self.issues = tuple(issues) + message = "; ".join( + f"{issue.code}{f'[{issue.path}]' if issue.path else ''}: {issue.reason}" + for issue in self.issues + ) + super().__init__(message) + + +@dataclass(frozen=True) +class ExperimentPlan: + """A deterministic plan plus non-fatal capability findings.""" + + definition: ExperimentDefinition + cases: tuple[ExperimentCase, ...] + issues: tuple[PlanningIssue, ...] = () + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": "cross_config.experiment_plan.v1", + "experiment_id": self.definition.experiment_id, + "cases": [case.to_dict() for case in self.cases], + "issues": [issue.to_dict() for issue in self.issues], + } + + +def _positive_int(value: Any) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError("must be a positive integer") + return value + + +def _strict_bool(value: Any) -> bool: + if not isinstance(value, bool): + raise ValueError("must be a JSON boolean") + return value + + +def _normalize_dtype(value: Any) -> str: + if not isinstance(value, str): + raise ValueError("must be a dtype string") + normalized = value.strip().lower().replace("torch.", "") + aliases = { + "bf16": "bfloat16", + "bfloat16": "bfloat16", + "fp16": "float16", + "half": "float16", + "float16": "float16", + "fp32": "float32", + "float": "float32", + "float32": "float32", + } + try: + return aliases[normalized] + except KeyError as exc: + raise ValueError(f"unsupported dtype {value!r}") from exc + + +def _normalize_choice(*choices: str) -> Normalizer: + allowed = frozenset(choices) + + def normalize(value: Any) -> str: + if not isinstance(value, str): + raise ValueError("must be a string") + normalized = value.strip().lower().replace("-", "_") + if normalized not in allowed: + raise ValueError(f"must be one of {sorted(allowed)}") + return normalized + + return normalize + + +def normalize_backend_id(value: Any) -> str: + """Normalize the public selected-logprob backend shortcut.""" + + if not isinstance(value, str) or not value.strip(): + raise ValueError("must be a non-empty backend ID") + normalized = value.strip().lower().replace("-", "_") + aliases = { + "auto": "native", + "default": "native", + "pytorch": "rlkernel.reference_logp", + "reference": "rlkernel.reference_logp", + } + return aliases.get(normalized, normalized) + + +V1_KNOB_DESCRIPTORS: tuple[KnobDescriptor, ...] = ( + KnobDescriptor("batch.size", IsolationScope.REQUEST, ("rollout", "training")), + KnobDescriptor("rollout.tensor_parallel_size", IsolationScope.PROCESS, ("rollout",)), + KnobDescriptor("rollout.context_parallel_size", IsolationScope.PROCESS, ("rollout",)), + KnobDescriptor("rollout.dtype", IsolationScope.ENGINE_CONSTRUCTION, ("rollout",)), + KnobDescriptor( + "rollout.enable_prefix_caching", + IsolationScope.ENGINE_CONSTRUCTION, + ("rollout",), + ), + KnobDescriptor("rollout.enforce_eager", IsolationScope.ENGINE_CONSTRUCTION, ("rollout",)), + KnobDescriptor( + "training.attention_backend", + IsolationScope.ENGINE_CONSTRUCTION, + ("training",), + allowed_values=("flash_attention_2", "sdpa", "eager", "model_default"), + ), + KnobDescriptor("training.compute_dtype", IsolationScope.ENGINE_CONSTRUCTION, ("training",)), + KnobDescriptor("logp.backend", IsolationScope.ENGINE_CONSTRUCTION, ("rollout", "training")), + KnobDescriptor( + "training.sharding", + IsolationScope.PROCESS, + ("training",), + allowed_values=("unsharded", "fsdp"), + ), +) + +V1_KNOBS: Mapping[str, KnobDescriptor] = { + descriptor.path: descriptor for descriptor in V1_KNOB_DESCRIPTORS +} + +_NORMALIZERS: Mapping[str, Normalizer] = { + "batch.size": _positive_int, + "rollout.tensor_parallel_size": _positive_int, + "rollout.context_parallel_size": _positive_int, + "rollout.dtype": _normalize_dtype, + "rollout.enable_prefix_caching": _strict_bool, + "rollout.enforce_eager": _strict_bool, + "training.attention_backend": _normalize_choice( + "flash_attention_2", "sdpa", "eager", "model_default" + ), + "training.compute_dtype": _normalize_dtype, + "logp.backend": normalize_backend_id, + "training.sharding": _normalize_choice("unsharded", "fsdp"), +} + + +class Planner: + """Generate a bounded plan without importing runtime- or operator-specific branches.""" + + def __init__( + self, + *, + knobs: Mapping[str, KnobDescriptor] = V1_KNOBS, + normalizers: Mapping[str, Normalizer] = _NORMALIZERS, + constraints: Sequence[Constraint] = (), + ): + self.knobs = dict(knobs) + self.normalizers = dict(normalizers) + self.constraints = tuple(constraints) + + def plan(self, definition: ExperimentDefinition) -> ExperimentPlan: + issues = self._validate_definition(definition) + if issues: + raise PlanningError(issues) + + baseline = self.normalize_requested(definition.baseline) + requested_cases: list[tuple[dict[str, Any], tuple[str, ...]]] = [(baseline, ())] + intervention_values: dict[str, tuple[Any, ...]] = {} + + for intervention in definition.interventions: + normalized_values = tuple( + self._normalize_value(intervention.path, value) for value in intervention.values + ) + intervention_values[intervention.path] = normalized_values + baseline_value = _get_path(baseline, intervention.path) + for value in normalized_values: + if value == baseline_value: + continue + requested = _deep_copy_mapping(baseline) + _set_path(requested, intervention.path, value) + requested_cases.append((requested, (intervention.path,))) + + if definition.strategy == PlanningStrategy.PAIRWISE: + for first_path, second_path in definition.pairwise_paths: + first_baseline = _get_path(baseline, first_path) + second_baseline = _get_path(baseline, second_path) + for first_value, second_value in itertools.product( + intervention_values[first_path], intervention_values[second_path] + ): + if first_value == first_baseline or second_value == second_baseline: + continue + requested = _deep_copy_mapping(baseline) + _set_path(requested, first_path, first_value) + _set_path(requested, second_path, second_value) + requested_cases.append((requested, tuple(sorted((first_path, second_path))))) + + contract_fingerprint = tolerance_contract_fingerprint() + scenario_fingerprint = _fingerprint( + {"scenario_id": definition.scenario_id, "scenario": definition.scenario} + ) + cases: list[ExperimentCase] = [] + seen_ids: set[str] = set() + capability_issues: list[PlanningIssue] = [] + for requested, changed_paths in requested_cases: + case_issues = self._apply_constraints(requested, changed_paths) + capability_issues.extend(case_issues) + case_id = self._case_id( + definition, + requested, + contract_fingerprint=contract_fingerprint, + scenario_fingerprint=scenario_fingerprint, + ) + if case_id in seen_ids: + continue + seen_ids.add(case_id) + cases.append( + ExperimentCase( + case_id=case_id, + experiment_id=definition.experiment_id, + scenario_id=definition.scenario_id, + identity=definition.identity, + requested=requested, + changed_paths=changed_paths, + contract_fingerprint=contract_fingerprint, + scenario_fingerprint=scenario_fingerprint, + ) + ) + + return ExperimentPlan( + definition=definition, + cases=tuple(cases), + issues=tuple(capability_issues), + ) + + def normalize_requested(self, requested: Mapping[str, Any]) -> dict[str, Any]: + flattened = _flatten(requested) + issues: list[PlanningIssue] = [] + normalized: dict[str, Any] = {} + for path, value in flattened.items(): + if path not in self.knobs: + code = "DERIVED_KNOB" if path == "logp.tp_layout" else "UNSUPPORTED_PATH" + issues.append( + PlanningIssue( + code=code, + path=path, + value=value, + reason="path is not a user-settable V1 knob", + ) + ) + continue + try: + normalized[path] = self._normalize_value(path, value) + except (TypeError, ValueError) as exc: + issues.append( + PlanningIssue( + code="UNSUPPORTED_VALUE", + path=path, + value=value, + reason=str(exc), + ) + ) + if issues: + raise PlanningError(issues) + result: dict[str, Any] = {} + for path, value in normalized.items(): + _set_path(result, path, value) + return result + + def isolation_for(self, changed_paths: Sequence[str]) -> IsolationScope: + if not changed_paths: + return IsolationScope.REQUEST + order = { + IsolationScope.REQUEST: 0, + IsolationScope.ENGINE_CONSTRUCTION: 1, + IsolationScope.DISTRIBUTED_CONTEXT: 2, + IsolationScope.PROCESS: 3, + } + return max((self.knobs[path].lifecycle for path in changed_paths), key=order.__getitem__) + + def _validate_definition(self, definition: ExperimentDefinition) -> list[PlanningIssue]: + issues: list[PlanningIssue] = [] + try: + baseline = self.normalize_requested(definition.baseline) + except PlanningError as exc: + return list(exc.issues) + baseline_paths = set(_flatten(baseline)) + for path in sorted(set(self.knobs).difference(baseline_paths)): + issues.append( + PlanningIssue( + code="MISSING_BASELINE_VALUE", + path=path, + reason="strict baselines must declare every allowlisted knob", + ) + ) + declared_paths: set[str] = set() + for intervention in definition.interventions: + path = intervention.path + if path not in self.knobs: + issues.append( + PlanningIssue( + code="UNSUPPORTED_PATH", + path=path, + reason="intervention path is not in the V1 allowlist", + ) + ) + continue + if path in declared_paths: + issues.append( + PlanningIssue( + code="DUPLICATE_INTERVENTION", + path=path, + reason="each intervention path must be declared once", + ) + ) + declared_paths.add(path) + if not intervention.values: + issues.append( + PlanningIssue( + code="EMPTY_INTERVENTION", + path=path, + reason="intervention values cannot be empty", + ) + ) + try: + _get_path(baseline, path) + except KeyError: + issues.append( + PlanningIssue( + code="MISSING_BASELINE_VALUE", + path=path, + reason="every intervention path must exist in baseline", + ) + ) + for value in intervention.values: + try: + self._normalize_value(path, value) + except (TypeError, ValueError) as exc: + issues.append( + PlanningIssue( + code="UNSUPPORTED_VALUE", + path=path, + value=value, + reason=str(exc), + ) + ) + + if definition.strategy == PlanningStrategy.ONE_AT_A_TIME and definition.pairwise_paths: + issues.append( + PlanningIssue( + code="PAIRWISE_NOT_ENABLED", + reason="pairwise_paths require strategy='pairwise'", + ) + ) + if definition.strategy == PlanningStrategy.PAIRWISE and not definition.pairwise_paths: + issues.append( + PlanningIssue( + code="PAIRWISE_PATHS_REQUIRED", + reason="pairwise strategy requires at least one explicit path pair", + ) + ) + seen_pairs: set[tuple[str, str]] = set() + for pair in definition.pairwise_paths: + if len(pair) != 2: + issues.append( + PlanningIssue( + code="INVALID_PAIR", + reason="each pairwise entry must contain exactly two paths", + value=pair, + ) + ) + continue + first, second = pair + canonical_pair = (first, second) if first < second else (second, first) + if first == second: + issues.append( + PlanningIssue( + code="INVALID_PAIR", + reason="pairwise paths must be distinct", + value=pair, + ) + ) + elif first not in declared_paths or second not in declared_paths: + issues.append( + PlanningIssue( + code="UNDECLARED_PAIR_PATH", + reason="pairwise paths must both have declared interventions", + value=pair, + ) + ) + elif canonical_pair in seen_pairs: + issues.append( + PlanningIssue( + code="DUPLICATE_PAIR", + reason="pairwise path pair is duplicated", + value=pair, + ) + ) + seen_pairs.add(canonical_pair) + return issues + + def _normalize_value(self, path: str, value: Any) -> Any: + try: + normalizer = self.normalizers[path] + except KeyError as exc: + raise ValueError(f"no normalizer registered for {path}") from exc + return normalizer(value) + + def _apply_constraints( + self, requested: Mapping[str, Any], changed_paths: Sequence[str] + ) -> list[PlanningIssue]: + issues: list[PlanningIssue] = [] + paths = changed_paths or tuple(_flatten(requested)) + for path in paths: + value = _get_path(requested, path) + for constraint in self.constraints: + issue = constraint(path, value, requested) + if issue is not None: + issues.append(issue) + return issues + + @staticmethod + def _case_id( + definition: ExperimentDefinition, + requested: Mapping[str, Any], + *, + contract_fingerprint: str, + scenario_fingerprint: str, + ) -> str: + payload = { + "requested": requested, + "identity": definition.identity.to_dict(), + "contract": { + "source": definition.contract_source, + "version": definition.contract_version, + "fingerprint": contract_fingerprint, + }, + "scenario_id": definition.scenario_id, + "scenario_fingerprint": scenario_fingerprint, + } + return f"cross-config-{_fingerprint(payload)[:20]}" + + +def _flatten(value: Mapping[str, Any], prefix: str = "") -> dict[str, Any]: + flattened: dict[str, Any] = {} + for key, child in value.items(): + if not isinstance(key, str) or not key: + raise PlanningError( + [PlanningIssue(code="INVALID_PATH", reason="configuration keys must be strings")] + ) + path = f"{prefix}.{key}" if prefix else key + if isinstance(child, Mapping): + flattened.update(_flatten(child, path)) + else: + flattened[path] = child + return flattened + + +def _get_path(value: Mapping[str, Any], path: str) -> Any: + current: Any = value + for part in path.split("."): + if not isinstance(current, Mapping) or part not in current: + raise KeyError(path) + current = current[part] + return current + + +def _set_path(value: dict[str, Any], path: str, child: Any) -> None: + current = value + parts = path.split(".") + for part in parts[:-1]: + existing = current.setdefault(part, {}) + if not isinstance(existing, dict): + raise ValueError(f"configuration path collision at {path}") + current = existing + current[parts[-1]] = child + + +def _deep_copy_mapping(value: Mapping[str, Any]) -> dict[str, Any]: + return json.loads(json.dumps(value)) + + +def _fingerprint(value: Mapping[str, Any]) -> str: + payload = json.dumps( + _json_plain(value), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def _json_plain(value: Any) -> Any: + if isinstance(value, Mapping): + return {str(key): _json_plain(child) for key, child in value.items()} + if isinstance(value, (tuple, list)): + return [_json_plain(child) for child in value] + return value + + +__all__ = [ + "ExperimentPlan", + "Planner", + "PlanningError", + "PlanningIssue", + "V1_KNOBS", + "V1_KNOB_DESCRIPTORS", + "normalize_backend_id", +] diff --git a/rl_engine/alignment/cross_config/schema.py b/rl_engine/alignment/cross_config/schema.py new file mode 100644 index 00000000..eb9d8e1d --- /dev/null +++ b/rl_engine/alignment/cross_config/schema.py @@ -0,0 +1,582 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Stable, versioned domain schema for cross-configuration alignment.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field, fields +from enum import Enum +from pathlib import Path +from types import MappingProxyType +from typing import Any, Mapping, Optional, Sequence + +import torch + + +class ScoreSide(str, Enum): + ROLLOUT = "rollout" + TRAINING = "training" + + +class IsolationScope(str, Enum): + REQUEST = "request" + ENGINE_CONSTRUCTION = "engine_construction" + DISTRIBUTED_CONTEXT = "distributed_context" + PROCESS = "process" + + +class PlanningStrategy(str, Enum): + ONE_AT_A_TIME = "one_at_a_time" + PAIRWISE = "pairwise" + + +class MaterializationStatus(str, Enum): + UNSUPPORTED = "unsupported" + APPLIED = "applied" + FALLBACK = "fallback" + UNOBSERVABLE = "unobservable" + ERROR = "error" + + +class AlignmentStatus(str, Enum): + PASS = "pass" + FAIL = "fail" + INVALID_IDENTITY = "invalid_identity" + INVALID_ARTIFACT = "invalid_artifact" + ZERO_ACTIVE_TOKENS = "zero_active_tokens" + + +class SerializableModel: + """Mixin providing a stable JSON-compatible representation.""" + + def to_dict(self) -> dict[str, Any]: + return { + item.name: _serialize_value(getattr(self, item.name)) + for item in fields(self) # type: ignore[arg-type] + } + + def to_json(self, *, indent: Optional[int] = None) -> str: + return json.dumps(self.to_dict(), indent=indent, sort_keys=True) + + +def _serialize_value(value: Any) -> Any: + if isinstance(value, Enum): + return value.value + if isinstance(value, SerializableModel): + return value.to_dict() + if isinstance(value, torch.Tensor): + snapshot = value.detach().cpu() + return { + "dtype": str(snapshot.dtype).replace("torch.", ""), + "shape": list(snapshot.shape), + "values": snapshot.tolist(), + } + if isinstance(value, Mapping): + return {str(key): _serialize_value(item) for key, item in value.items()} + if isinstance(value, (tuple, list)): + return [_serialize_value(item) for item in value] + if isinstance(value, (set, frozenset)): + return [_serialize_value(item) for item in sorted(value, key=repr)] + if isinstance(value, Path): + return str(value) + if isinstance(value, torch.dtype): + return str(value).replace("torch.", "") + return value + + +def _freeze_value(value: Any) -> Any: + if isinstance(value, Mapping): + return MappingProxyType({str(key): _freeze_value(item) for key, item in value.items()}) + if isinstance(value, (tuple, list)): + return tuple(_freeze_value(item) for item in value) + if isinstance(value, (set, frozenset)): + return tuple(sorted((_freeze_value(item) for item in value), key=repr)) + return value + + +def _freeze_mapping(value: Mapping[str, Any]) -> Mapping[str, Any]: + return _freeze_value(value) + + +def _coerce_enum(value: Any, enum_type: type[Enum]) -> Enum: + if isinstance(value, enum_type): + return value + return enum_type(value) + + +def _int_matrix(value: Sequence[Sequence[int]]) -> tuple[tuple[int, ...], ...]: + rows: list[tuple[int, ...]] = [] + for row in value: + normalized: list[int] = [] + for item in row: + if isinstance(item, bool) or not isinstance(item, int): + raise ValueError("integer identity matrices accept JSON integers only") + normalized.append(item) + rows.append(tuple(normalized)) + return tuple(rows) + + +def _bool_matrix(value: Sequence[Sequence[bool]]) -> tuple[tuple[bool, ...], ...]: + rows: list[tuple[bool, ...]] = [] + for row in value: + normalized: list[bool] = [] + for item in row: + if not isinstance(item, bool): + raise ValueError("boolean identity matrices accept JSON booleans only") + normalized.append(item) + rows.append(tuple(normalized)) + return tuple(rows) + + +def _validate_rectangular(name: str, value: tuple[tuple[Any, ...], ...]) -> None: + if not value: + return + width = len(value[0]) + if any(len(row) != width for row in value): + raise ValueError(f"{name} must be rectangular") + + +def _validate_same_matrix_shape( + left_name: str, + left: tuple[tuple[Any, ...], ...], + right_name: str, + right: tuple[tuple[Any, ...], ...], +) -> None: + if left and right and (len(left), len(left[0])) != (len(right), len(right[0])): + raise ValueError(f"{left_name} shape must match {right_name} shape") + + +def _snapshot_tensor(value: torch.Tensor, *, dtype: Optional[torch.dtype] = None) -> torch.Tensor: + if not isinstance(value, torch.Tensor): + raise TypeError(f"expected torch.Tensor, got {type(value)!r}") + snapshot = value.detach().clone() + return snapshot.to(dtype=dtype) if dtype is not None else snapshot + + +@dataclass(frozen=True) +class SemanticIdentitySpec(SerializableModel): + """Logical inputs that must match before numerical comparison is meaningful.""" + + checkpoint_id: str + model_version: str + tokenizer_policy: str + token_ids: tuple[tuple[int, ...], ...] + selected_token_ids: tuple[tuple[int, ...], ...] + active_mask: tuple[tuple[bool, ...], ...] + pre_update_state: str + tokenizer_id: str = "" + attention_mask: tuple[tuple[bool, ...], ...] = () + position_ids: tuple[tuple[int, ...], ...] = () + cache_metadata: Mapping[str, Any] = field(default_factory=dict) + packing_metadata: Mapping[str, Any] = field(default_factory=dict) + schema_version: str = "cross_config.semantic_identity.v1" + + def __post_init__(self) -> None: + if self.schema_version != "cross_config.semantic_identity.v1": + raise ValueError("unsupported SemanticIdentitySpec schema_version") + if not self.checkpoint_id: + raise ValueError("checkpoint_id must not be empty") + if not self.model_version: + raise ValueError("model_version must not be empty") + if not self.tokenizer_policy: + raise ValueError("tokenizer_policy must not be empty") + if not self.pre_update_state: + raise ValueError("pre_update_state must not be empty") + + object.__setattr__(self, "token_ids", _int_matrix(self.token_ids)) + object.__setattr__(self, "selected_token_ids", _int_matrix(self.selected_token_ids)) + object.__setattr__(self, "active_mask", _bool_matrix(self.active_mask)) + object.__setattr__(self, "attention_mask", _bool_matrix(self.attention_mask)) + object.__setattr__(self, "position_ids", _int_matrix(self.position_ids)) + object.__setattr__(self, "cache_metadata", _freeze_mapping(self.cache_metadata)) + object.__setattr__(self, "packing_metadata", _freeze_mapping(self.packing_metadata)) + + if not self.token_ids or not self.token_ids[0]: + raise ValueError("token_ids must contain at least one token") + if not self.selected_token_ids: + raise ValueError("selected_token_ids must not be empty") + if not self.active_mask: + raise ValueError("active_mask must not be empty") + if not self.attention_mask: + raise ValueError("attention_mask must not be empty") + for name in ( + "token_ids", + "selected_token_ids", + "active_mask", + "attention_mask", + "position_ids", + ): + _validate_rectangular(name, getattr(self, name)) + _validate_same_matrix_shape( + "token_ids", + self.token_ids, + "selected_token_ids", + self.selected_token_ids, + ) + _validate_same_matrix_shape( + "selected_token_ids", self.selected_token_ids, "active_mask", self.active_mask + ) + _validate_same_matrix_shape( + "token_ids", self.token_ids, "attention_mask", self.attention_mask + ) + _validate_same_matrix_shape("token_ids", self.token_ids, "position_ids", self.position_ids) + + +@dataclass(frozen=True) +class ScorerSpec(SerializableModel): + side: ScoreSide + backend_id: str + dtype: str + device: str = "cpu" + world_size: int = 1 + topology: Mapping[str, Any] = field(default_factory=dict) + construction_options: Mapping[str, Any] = field(default_factory=dict) + operator_overrides: Mapping[str, str] = field(default_factory=dict) + schema_version: str = "cross_config.scorer.v1" + + def __post_init__(self) -> None: + object.__setattr__(self, "side", _coerce_enum(self.side, ScoreSide)) + if not self.backend_id: + raise ValueError("backend_id must not be empty") + if not self.dtype: + raise ValueError("dtype must not be empty") + if self.world_size < 1: + raise ValueError("world_size must be >= 1") + object.__setattr__(self, "topology", _freeze_mapping(self.topology)) + object.__setattr__(self, "construction_options", _freeze_mapping(self.construction_options)) + object.__setattr__(self, "operator_overrides", _freeze_mapping(self.operator_overrides)) + + +@dataclass(frozen=True) +class KnobDescriptor(SerializableModel): + path: str + lifecycle: IsolationScope + targets: tuple[str, ...] + allowed_values: tuple[Any, ...] = () + derived: bool = False + critical: bool = True + schema_version: str = "cross_config.knob_descriptor.v1" + + def __post_init__(self) -> None: + object.__setattr__(self, "lifecycle", _coerce_enum(self.lifecycle, IsolationScope)) + object.__setattr__(self, "targets", tuple(str(target) for target in self.targets)) + object.__setattr__(self, "allowed_values", tuple(_freeze_value(self.allowed_values))) + if not self.path: + raise ValueError("path must not be empty") + if not self.targets: + raise ValueError("targets must not be empty") + + +@dataclass(frozen=True) +class InterventionSpec(SerializableModel): + path: str + values: tuple[Any, ...] + schema_version: str = "cross_config.intervention.v1" + + def __post_init__(self) -> None: + if not self.path: + raise ValueError("path must not be empty") + object.__setattr__(self, "values", tuple(_freeze_value(self.values))) + if not self.values: + raise ValueError("values must not be empty") + + +@dataclass(frozen=True) +class ExperimentDefinition(SerializableModel): + experiment_id: str + scenario_id: str + identity: SemanticIdentitySpec + baseline: Mapping[str, Any] + interventions: tuple[InterventionSpec, ...] = () + scenario: Mapping[str, Any] = field(default_factory=dict) + strategy: PlanningStrategy = PlanningStrategy.ONE_AT_A_TIME + strict_fallback: bool = True + pairwise_paths: tuple[tuple[str, str], ...] = () + contract_source: str = "ws1" + contract_version: str = "current" + schema_version: str = "cross_config.experiment_definition.v1" + + def __post_init__(self) -> None: + if not self.experiment_id: + raise ValueError("experiment_id must not be empty") + if not self.scenario_id: + raise ValueError("scenario_id must not be empty") + if self.contract_source != "ws1": + raise ValueError("Cross-configuration alignment V1 requires contract_source='ws1'") + if self.contract_version != "current": + raise ValueError("Cross-configuration alignment V1 requires contract_version='current'") + object.__setattr__(self, "baseline", _freeze_mapping(self.baseline)) + object.__setattr__(self, "scenario", _freeze_mapping(self.scenario)) + object.__setattr__(self, "interventions", tuple(self.interventions)) + object.__setattr__(self, "strategy", _coerce_enum(self.strategy, PlanningStrategy)) + normalized_pairs: list[tuple[str, str]] = [] + for pair in self.pairwise_paths: + if len(pair) != 2: + raise ValueError("each pairwise_paths entry must contain exactly two paths") + normalized_pairs.append((str(pair[0]), str(pair[1]))) + object.__setattr__(self, "pairwise_paths", tuple(normalized_pairs)) + + +@dataclass(frozen=True) +class ExperimentCase(SerializableModel): + case_id: str + experiment_id: str + scenario_id: str + identity: SemanticIdentitySpec + requested: Mapping[str, Any] + execution_binding: Mapping[str, Any] = field(default_factory=dict) + changed_paths: tuple[str, ...] = () + contract_fingerprint: str = "" + scenario_fingerprint: str = "" + schema_version: str = "cross_config.experiment_case.v1" + + def __post_init__(self) -> None: + if not self.case_id: + raise ValueError("case_id must not be empty") + if not self.experiment_id: + raise ValueError("experiment_id must not be empty") + if not self.scenario_id: + raise ValueError("scenario_id must not be empty") + object.__setattr__(self, "requested", _freeze_mapping(self.requested)) + object.__setattr__(self, "execution_binding", _freeze_mapping(self.execution_binding)) + object.__setattr__(self, "changed_paths", tuple(str(path) for path in self.changed_paths)) + + +@dataclass(frozen=True) +class MaterializedCase(SerializableModel): + case: ExperimentCase + normalized: Mapping[str, Any] + materialized: Mapping[str, Any] + isolation_scope: IsolationScope + construction_fingerprint: str = "" + distributed_context_fingerprint: str = "" + process_fingerprint: str = "" + status: MaterializationStatus = MaterializationStatus.APPLIED + evidence: Mapping[str, Any] = field(default_factory=dict) + schema_version: str = "cross_config.materialized_case.v1" + + def __post_init__(self) -> None: + object.__setattr__(self, "normalized", _freeze_mapping(self.normalized)) + object.__setattr__(self, "materialized", _freeze_mapping(self.materialized)) + object.__setattr__( + self, + "isolation_scope", + _coerce_enum(self.isolation_scope, IsolationScope), + ) + object.__setattr__(self, "status", _coerce_enum(self.status, MaterializationStatus)) + object.__setattr__(self, "evidence", _freeze_mapping(self.evidence)) + + +@dataclass(frozen=True) +class CanonicalScoringBatch(SerializableModel): + identity: SemanticIdentitySpec + input_ids: torch.Tensor + selected_token_ids: torch.Tensor + active_mask: torch.Tensor + attention_mask: torch.Tensor + position_ids: Optional[torch.Tensor] = None + metadata: Mapping[str, Any] = field(default_factory=dict) + schema_version: str = "cross_config.canonical_scoring_batch.v1" + + def __post_init__(self) -> None: + object.__setattr__(self, "input_ids", _snapshot_tensor(self.input_ids, dtype=torch.long)) + object.__setattr__( + self, "selected_token_ids", _snapshot_tensor(self.selected_token_ids, dtype=torch.long) + ) + object.__setattr__( + self, "active_mask", _snapshot_tensor(self.active_mask, dtype=torch.bool) + ) + object.__setattr__( + self, "attention_mask", _snapshot_tensor(self.attention_mask, dtype=torch.bool) + ) + if self.position_ids is not None: + object.__setattr__( + self, "position_ids", _snapshot_tensor(self.position_ids, dtype=torch.long) + ) + object.__setattr__(self, "metadata", _freeze_mapping(self.metadata)) + + shape = self.input_ids.shape + if self.input_ids.ndim != 2: + raise ValueError("input_ids must have shape [batch, sequence]") + for name in ("selected_token_ids", "active_mask", "attention_mask"): + if getattr(self, name).shape != shape: + raise ValueError(f"{name} shape must match input_ids shape") + if self.position_ids is not None and self.position_ids.shape != shape: + raise ValueError("position_ids shape must match input_ids shape") + + _require_tensor_matches_matrix("input_ids", self.input_ids, self.identity.token_ids) + _require_tensor_matches_matrix( + "selected_token_ids", self.selected_token_ids, self.identity.selected_token_ids + ) + _require_tensor_matches_matrix("active_mask", self.active_mask, self.identity.active_mask) + _require_tensor_matches_matrix( + "attention_mask", self.attention_mask, self.identity.attention_mask + ) + if self.identity.position_ids: + if self.position_ids is None: + raise ValueError("position_ids are required by the semantic identity") + _require_tensor_matches_matrix( + "position_ids", self.position_ids, self.identity.position_ids + ) + elif self.position_ids is not None: + raise ValueError("position_ids were supplied but are absent from semantic identity") + + +def _require_tensor_matches_matrix( + name: str, + tensor: torch.Tensor, + matrix: tuple[tuple[Any, ...], ...], +) -> None: + expected = torch.tensor(matrix, dtype=tensor.dtype, device=tensor.device) + if expected.shape != tensor.shape or not torch.equal(tensor, expected): + raise ValueError(f"{name} does not match the semantic identity") + + +@dataclass(frozen=True) +class RuntimeProvenance(SerializableModel): + requested: Mapping[str, Any] + normalized: Mapping[str, Any] + materialized: Mapping[str, Any] + actual: Mapping[str, Any] + status: MaterializationStatus = MaterializationStatus.APPLIED + construction_fingerprint: str = "" + distributed_context_fingerprint: str = "" + process_fingerprint: str = "" + implementation_fingerprint: str = "" + evidence: Mapping[str, Any] = field(default_factory=dict) + rank: int = 0 + world_size: int = 1 + schema_version: str = "cross_config.runtime_provenance.v1" + + def __post_init__(self) -> None: + object.__setattr__(self, "requested", _freeze_mapping(self.requested)) + object.__setattr__(self, "normalized", _freeze_mapping(self.normalized)) + object.__setattr__(self, "materialized", _freeze_mapping(self.materialized)) + object.__setattr__(self, "actual", _freeze_mapping(self.actual)) + object.__setattr__(self, "status", _coerce_enum(self.status, MaterializationStatus)) + object.__setattr__(self, "evidence", _freeze_mapping(self.evidence)) + if self.rank < 0: + raise ValueError("rank must be >= 0") + if self.world_size < 1: + raise ValueError("world_size must be >= 1") + if self.rank >= self.world_size: + raise ValueError("rank must be less than world_size") + + +@dataclass(frozen=True) +class ScoreArtifact(SerializableModel): + case_id: str + attempt_id: str + side: ScoreSide + identity: SemanticIdentitySpec + scorer: ScorerSpec + selected_logprobs: torch.Tensor + active_mask: torch.Tensor + provenance: RuntimeProvenance + schema_version: str = "cross_config.score_artifact.v1" + + def __post_init__(self) -> None: + object.__setattr__(self, "side", _coerce_enum(self.side, ScoreSide)) + if not self.case_id: + raise ValueError("case_id must not be empty") + if not self.attempt_id: + raise ValueError("attempt_id must not be empty") + if self.scorer.side is not self.side: + raise ValueError("scorer side must match score artifact side") + object.__setattr__(self, "selected_logprobs", _snapshot_tensor(self.selected_logprobs)) + object.__setattr__( + self, "active_mask", _snapshot_tensor(self.active_mask, dtype=torch.bool) + ) + if self.selected_logprobs.shape != self.active_mask.shape: + raise ValueError("selected_logprobs shape must match active_mask shape") + + +@dataclass(frozen=True) +class TokenComparisonArtifact(SerializableModel): + rollout_logprobs: torch.Tensor + training_logprobs: torch.Tensor + active_mask: torch.Tensor + absolute_diff: torch.Tensor + mismatch_mask: torch.Tensor + fixed_threshold: float + schema_version: str = "cross_config.token_comparison.v1" + + def __post_init__(self) -> None: + object.__setattr__(self, "rollout_logprobs", _snapshot_tensor(self.rollout_logprobs)) + object.__setattr__(self, "training_logprobs", _snapshot_tensor(self.training_logprobs)) + object.__setattr__( + self, "active_mask", _snapshot_tensor(self.active_mask, dtype=torch.bool) + ) + object.__setattr__(self, "absolute_diff", _snapshot_tensor(self.absolute_diff)) + object.__setattr__( + self, "mismatch_mask", _snapshot_tensor(self.mismatch_mask, dtype=torch.bool) + ) + shape = self.rollout_logprobs.shape + for name in ( + "training_logprobs", + "active_mask", + "absolute_diff", + "mismatch_mask", + ): + if getattr(self, name).shape != shape: + raise ValueError(f"{name} shape must match rollout_logprobs shape") + if self.fixed_threshold < 0.0: + raise ValueError("fixed_threshold must be non-negative") + + +@dataclass(frozen=True) +class AlignmentResult(SerializableModel): + case_id: str + attempt_id: str + status: AlignmentStatus + comparable: bool + passed: bool + active_token_count: int + mismatch_count: int + contract_fingerprint: str + fixed_threshold: Optional[float] = None + identity_errors: tuple[str, ...] = () + artifact_errors: tuple[str, ...] = () + diagnostics: Mapping[str, Any] = field(default_factory=dict) + token_artifact: Optional[TokenComparisonArtifact] = None + schema_version: str = "cross_config.alignment_result.v1" + + def __post_init__(self) -> None: + object.__setattr__(self, "status", _coerce_enum(self.status, AlignmentStatus)) + object.__setattr__(self, "identity_errors", tuple(self.identity_errors)) + object.__setattr__(self, "artifact_errors", tuple(self.artifact_errors)) + object.__setattr__(self, "diagnostics", _freeze_mapping(self.diagnostics)) + if self.active_token_count < 0: + raise ValueError("active_token_count must be non-negative") + if self.mismatch_count < 0: + raise ValueError("mismatch_count must be non-negative") + if self.mismatch_count > self.active_token_count: + raise ValueError("mismatch_count cannot exceed active_token_count") + if self.status is AlignmentStatus.PASS and not self.passed: + raise ValueError("PASS result must set passed=True") + if self.status is not AlignmentStatus.PASS and self.passed: + raise ValueError("only PASS results may set passed=True") + + +__all__ = [ + "AlignmentResult", + "AlignmentStatus", + "CanonicalScoringBatch", + "ExperimentCase", + "ExperimentDefinition", + "InterventionSpec", + "IsolationScope", + "KnobDescriptor", + "MaterializationStatus", + "MaterializedCase", + "PlanningStrategy", + "RuntimeProvenance", + "ScoreArtifact", + "ScoreSide", + "ScorerSpec", + "SemanticIdentitySpec", + "SerializableModel", + "TokenComparisonArtifact", +] diff --git a/rl_engine/kernels/gtest/tolerance.py b/rl_engine/kernels/gtest/tolerance.py index d0481e83..9b357059 100644 --- a/rl_engine/kernels/gtest/tolerance.py +++ b/rl_engine/kernels/gtest/tolerance.py @@ -3,6 +3,7 @@ from __future__ import annotations +import hashlib import json from pathlib import Path from typing import Any @@ -17,4 +18,58 @@ def load_contract(path: str | Path = _CONTRACT_PATH) -> dict[str, Any]: return json.load(handle) -__all__ = ["load_contract"] +def resolve_logprob_threshold(dtype: Any) -> float: + """Return the fixed WS1 selected-logprob absolute-difference threshold. + + The contract path is intentionally not configurable through this accessor. + Cross-configuration experiment definitions may select a dtype, but they cannot + inject or override a numerical threshold. + """ + + dtype_name = _normalize_dtype_name(dtype) + contract = load_contract() + try: + values = contract["accuracy"]["default"]["logprob"][dtype_name] + return float(values["atol"]) + except KeyError as exc: + raise ValueError(f"WS1 has no logprob threshold for dtype {dtype_name!r}") from exc + + +def tolerance_contract_fingerprint() -> str: + """Return a deterministic fingerprint of the current WS1 contract contents.""" + + canonical = json.dumps( + load_contract(), + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + return hashlib.sha256(canonical).hexdigest() + + +def _normalize_dtype_name(dtype: Any) -> str: + normalized = str(dtype).strip().lower().replace("torch.", "").replace("-", "") + aliases = { + "bf16": "bfloat16", + "bfloat16": "bfloat16", + "fp16": "float16", + "float16": "float16", + "half": "float16", + "fp32": "float32", + "float32": "float32", + "float": "float32", + } + try: + return aliases[normalized] + except KeyError as exc: + valid = ", ".join(sorted(set(aliases.values()))) + raise ValueError( + f"unsupported WS1 logprob dtype {dtype!r}; expected one of: {valid}" + ) from exc + + +__all__ = [ + "load_contract", + "resolve_logprob_threshold", + "tolerance_contract_fingerprint", +] From 4bda43f1e68c6ee9a97d5e55984be60001d4e8f1 Mon Sep 17 00:00:00 2001 From: CyberSecurityErial <2710555967@qq.com> Date: Sun, 19 Jul 2026 08:31:43 +0800 Subject: [PATCH 3/8] feat(alignment): add runtime materialization and scoring bridge --- rl_engine/alignment/cross_config/operators.py | 265 ++++++++++ rl_engine/alignment/cross_config/runtime.py | 453 ++++++++++++++++++ rl_engine/executors/stateless_executor.py | 44 +- 3 files changed, 758 insertions(+), 4 deletions(-) create mode 100644 rl_engine/alignment/cross_config/operators.py create mode 100644 rl_engine/alignment/cross_config/runtime.py diff --git a/rl_engine/alignment/cross_config/operators.py b/rl_engine/alignment/cross_config/operators.py new file mode 100644 index 00000000..e3105a28 --- /dev/null +++ b/rl_engine/alignment/cross_config/operators.py @@ -0,0 +1,265 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Target-specific semantic operator selection for alignment cases.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass, replace +from typing import Any, Literal, Mapping, Optional, cast + +import torch + +from rl_engine.kernels.semantic_registry import ( + OperatorInstanceProvenance, + OperatorRequirements, + OperatorResolution, + OperatorResolutionPolicy, + OperatorSession, + SemanticOperatorCatalog, +) + +OperatorTarget = Literal["rollout", "training", "both"] +ConcreteOperatorTarget = Literal["rollout", "training"] + + +@dataclass(frozen=True) +class OperatorOverride: + """Backend overrides for one semantic operator on either or both sides.""" + + semantic_op: str + rollout_backend: Optional[str] = None + training_backend: Optional[str] = None + + def __post_init__(self) -> None: + semantic_op = self.semantic_op.strip() + if not semantic_op: + raise ValueError("semantic_op must not be empty") + rollout_backend = _normalized_optional_backend(self.rollout_backend) + training_backend = _normalized_optional_backend(self.training_backend) + if rollout_backend is None and training_backend is None: + raise ValueError("operator override must select rollout, training, or both") + object.__setattr__(self, "semantic_op", semantic_op) + object.__setattr__(self, "rollout_backend", rollout_backend) + object.__setattr__(self, "training_backend", training_backend) + + @classmethod + def for_target( + cls, + *, + semantic_op: str, + backend_id: str, + target: OperatorTarget, + ) -> OperatorOverride: + """Create a rollout-only, training-only, or dual-side override.""" + + normalized_target = target.strip().lower() + if normalized_target == "rollout": + return cls(semantic_op=semantic_op, rollout_backend=backend_id) + if normalized_target == "training": + return cls(semantic_op=semantic_op, training_backend=backend_id) + if normalized_target == "both": + return cls( + semantic_op=semantic_op, + rollout_backend=backend_id, + training_backend=backend_id, + ) + raise ValueError("target must be 'rollout', 'training', or 'both'") + + def backend_for(self, target: ConcreteOperatorTarget) -> Optional[str]: + normalized_target = _concrete_target(target) + if normalized_target == "rollout": + return self.rollout_backend + return self.training_backend + + def to_dict(self) -> dict[str, Any]: + return { + "semantic_op": self.semantic_op, + "rollout_backend": self.rollout_backend, + "training_backend": self.training_backend, + } + + +@dataclass(frozen=True) +class ResolvedOperatorOverride: + """Target-specific exact resolutions produced from an operator override.""" + + semantic_op: str + rollout: Optional[OperatorResolution] = None + training: Optional[OperatorResolution] = None + + def for_target(self, target: ConcreteOperatorTarget) -> Optional[OperatorResolution]: + normalized_target = _concrete_target(target) + return self.rollout if normalized_target == "rollout" else self.training + + def to_dict(self) -> dict[str, Any]: + return { + "semantic_op": self.semantic_op, + "rollout": None if self.rollout is None else self.rollout.to_dict(), + "training": None if self.training is None else self.training.to_dict(), + } + + +class OperatorBridge: + """Resolve and instantiate semantic operator overrides without planner branches.""" + + def __init__( + self, + catalog: Optional[SemanticOperatorCatalog | OperatorSession] = None, + *, + policy: Optional[OperatorResolutionPolicy] = None, + ): + """Create a bridge backed by one case-local operator session.""" + + if isinstance(catalog, OperatorSession): + self.catalog = catalog.catalog + self.session = catalog + self.policy = policy or catalog.policy + else: + if catalog is None: + # Built-in descriptors are repository integration details; the + # generic semantic catalog itself remains backend-neutral. + from rl_engine.kernels.registry import kernel_registry + + catalog = kernel_registry.semantic + if not isinstance(catalog, SemanticOperatorCatalog): + raise TypeError("catalog must be a SemanticOperatorCatalog or OperatorSession") + self.catalog = catalog + self.policy = policy or OperatorResolutionPolicy() + self.session = self.catalog.session(self.policy) + + def resolve_override( + self, + override: OperatorOverride, + *, + requirements: Mapping[str, OperatorRequirements], + strict: bool = True, + ) -> ResolvedOperatorOverride: + """Resolve only the sides explicitly selected by ``override``.""" + + resolved: dict[ConcreteOperatorTarget, OperatorResolution] = {} + targets: tuple[ConcreteOperatorTarget, ...] = ("rollout", "training") + for target in targets: + backend_id = override.backend_for(target) + if backend_id is None: + continue + target_requirements = requirements.get(target) + if target_requirements is None: + raise ValueError(f"missing operator requirements for target {target!r}") + target_policy = replace(self.policy, strict=strict) + resolved[target] = self.session.resolve( + semantic_op=override.semantic_op, + requested_backend=backend_id, + target=target, + requirements=target_requirements, + policy=target_policy, + ) + return ResolvedOperatorOverride( + semantic_op=override.semantic_op, + rollout=resolved.get("rollout"), + training=resolved.get("training"), + ) + + def instantiate( + self, + resolved: ResolvedOperatorOverride, + *, + target: ConcreteOperatorTarget, + factory_kwargs: Optional[Mapping[str, Any]] = None, + cache: bool = False, + ) -> Any: + """Instantiate one resolved side; rollout and training remain independent.""" + + resolution = resolved.for_target(target) + if resolution is None: + raise ValueError(f"operator override does not select target {target!r}") + return self.session.instantiate( + resolution, + factory_kwargs=factory_kwargs, + cache=cache, + ) + + def instance_provenance( + self, + resolved: ResolvedOperatorOverride, + *, + target: ConcreteOperatorTarget, + instance: Any, + ) -> OperatorInstanceProvenance: + resolution = resolved.for_target(target) + if resolution is None: + raise ValueError(f"operator override does not select target {target!r}") + return self.session.instance_provenance(resolution, instance) + + +def selected_logprobs_with_operator( + operator: Any, + logits: torch.Tensor, + token_ids: torch.Tensor, + *, + active_mask: Optional[torch.Tensor] = None, + temperature: float = 1.0, + output_dtype: torch.dtype = torch.float32, +) -> torch.Tensor: + """Apply the repository selected-logprob interface with common mask semantics.""" + + if not math.isfinite(temperature) or temperature <= 0.0: + raise ValueError("temperature must be finite and greater than zero") + if logits.shape[:-1] != token_ids.shape: + raise ValueError( + f"logits leading shape {tuple(logits.shape[:-1])} must match " + f"token_ids shape {tuple(token_ids.shape)}" + ) + mask: Optional[torch.Tensor] = None + safe_token_ids = token_ids.to(device=logits.device, dtype=torch.long) + if active_mask is not None: + if active_mask.shape != token_ids.shape: + raise ValueError("active_mask shape must match token_ids shape") + mask = active_mask.to(device=logits.device, dtype=torch.bool) + safe_token_ids = safe_token_ids.masked_fill(~mask, 0) + + scaled_logits = logits.float() / float(temperature) + if hasattr(operator, "apply_fp32") and callable(operator.apply_fp32): + selected = operator.apply_fp32(scaled_logits, safe_token_ids) + elif callable(operator): + selected = operator(scaled_logits, safe_token_ids) + else: + raise TypeError("selected-logprob operator must be callable or expose apply_fp32") + if not isinstance(selected, torch.Tensor): + raise TypeError("selected-logprob operator must return a torch.Tensor") + if selected.shape != token_ids.shape: + raise ValueError( + f"selected-logprob output shape {tuple(selected.shape)} must match " + f"token_ids shape {tuple(token_ids.shape)}" + ) + selected = selected.to(device=logits.device, dtype=output_dtype) + if mask is not None: + selected = selected.masked_fill(~mask, 0.0) + return selected + + +def _normalized_optional_backend(value: Optional[str]) -> Optional[str]: + if value is None: + return None + normalized = value.strip() + if not normalized: + raise ValueError("backend_id must not be empty") + return normalized + + +def _concrete_target(target: ConcreteOperatorTarget) -> ConcreteOperatorTarget: + normalized = target.strip().lower() + if normalized not in {"rollout", "training"}: + raise ValueError("target must be 'rollout' or 'training'") + return cast(ConcreteOperatorTarget, normalized) + + +__all__ = [ + "ConcreteOperatorTarget", + "OperatorBridge", + "OperatorOverride", + "OperatorTarget", + "ResolvedOperatorOverride", + "selected_logprobs_with_operator", +] diff --git a/rl_engine/alignment/cross_config/runtime.py b/rl_engine/alignment/cross_config/runtime.py new file mode 100644 index 00000000..00f81f2a --- /dev/null +++ b/rl_engine/alignment/cross_config/runtime.py @@ -0,0 +1,453 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Thin runtime materialization facade for the V1 allowlist.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import Any, Iterable, Mapping, Protocol, Sequence + +from rl_engine.alignment.cross_config.planner import V1_KNOBS +from rl_engine.alignment.cross_config.schema import ( + ExperimentCase, + IsolationScope, + KnobDescriptor, + MaterializationStatus, + MaterializedCase, + RuntimeProvenance, + SerializableModel, +) + + +@dataclass(frozen=True) +class KnobApplication(SerializableModel): + """One adapter's requested, materialized, and observed value.""" + + path: str + requested: Any + materialized: Any + actual: Any + lifecycle: IsolationScope + status: MaterializationStatus + evidence: Mapping[str, Any] = field(default_factory=dict) + critical: bool = True + schema_version: str = "cross_config.knob_application.v1" + + def __post_init__(self) -> None: + object.__setattr__(self, "lifecycle", IsolationScope(self.lifecycle)) + object.__setattr__(self, "status", MaterializationStatus(self.status)) + for name in ("requested", "materialized", "actual"): + object.__setattr__(self, name, _freeze_value(getattr(self, name))) + object.__setattr__(self, "evidence", _freeze_mapping(self.evidence)) + + +@dataclass(frozen=True) +class RuntimeBinding: + """Small, backend-neutral handoff from materialization to execution. + + Runtime adapters may construct repository-specific objects internally, but + the core runner sees only the values required to create scorers and validate + lifecycle identity. New vLLM, FSDP, or other adapters therefore do not + change the runner's type surface. + """ + + batch_size: int + side_configs: Mapping[str, Mapping[str, Any]] + topology: Mapping[str, Mapping[str, Any]] + scorer: Mapping[str, Any] + operator_backends: Mapping[str, str] + runtime_kind: str + + def __post_init__(self) -> None: + if isinstance(self.batch_size, bool) or not isinstance(self.batch_size, int): + raise TypeError("batch_size must be an integer") + if self.batch_size < 1: + raise ValueError("batch_size must be greater than zero") + if not isinstance(self.runtime_kind, str) or not self.runtime_kind.strip(): + raise ValueError("runtime_kind must be a non-empty string") + for name, value in ( + ("side_configs", self.side_configs), + ("topology", self.topology), + ): + for target in ("rollout", "training"): + if not isinstance(value.get(target), Mapping): + raise ValueError(f"{name} must define a {target} mapping") + for target in ("rollout", "training"): + world_size = self.topology[target].get("world_size") + if isinstance(world_size, bool) or not isinstance(world_size, int) or world_size < 1: + raise ValueError(f"{target} topology must define a positive integer world_size") + backend = self.operator_backends.get(target) + if not isinstance(backend, str) or not backend.strip(): + raise ValueError(f"operator_backends must define a non-empty {target} backend") + object.__setattr__(self, "side_configs", _freeze_mapping(self.side_configs)) + object.__setattr__(self, "topology", _freeze_mapping(self.topology)) + object.__setattr__(self, "scorer", _freeze_mapping(self.scorer)) + object.__setattr__(self, "operator_backends", _freeze_mapping(self.operator_backends)) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": "cross_config.runtime_binding.v1", + "runtime_kind": self.runtime_kind, + "batch_size": self.batch_size, + "side_configs": _plain_mapping(self.side_configs), + "topology": _plain_mapping(self.topology), + "scorer": _plain_mapping(self.scorer), + "operators": dict(self.operator_backends), + } + + +@dataclass(frozen=True) +class AdapterMaterialization: + """Output of a typed runtime adapter before the facade adds fingerprints.""" + + applications: tuple[KnobApplication, ...] + binding: RuntimeBinding + + def __post_init__(self) -> None: + object.__setattr__(self, "applications", tuple(self.applications)) + + +class RuntimeMaterializer(Protocol): + """Adapter boundary used by the small ``RuntimeTools`` facade. + + The declared implementation fingerprint must deterministically identify the + executable materialization path and change when that implementation changes. + """ + + runtime_kind: str + + @property + def implementation_fingerprint(self) -> str: ... + + def materialize( + self, + normalized: Mapping[str, Any], + descriptors: Mapping[str, KnobDescriptor], + ) -> AdapterMaterialization: ... + + +@dataclass(frozen=True) +class RuntimeMaterialization: + materialized_case: MaterializedCase + provenance: RuntimeProvenance + applications: tuple[KnobApplication, ...] + binding: RuntimeBinding + + @property + def executable_in_strict_mode(self) -> bool: + return self.materialized_case.status is MaterializationStatus.APPLIED + + +class RuntimeMaterializationError(RuntimeError): + pass + + +class RuntimeTools: + """Materialize cases and compute reuse fingerprints without owning execution.""" + + def __init__(self, descriptors: Mapping[str, KnobDescriptor] = V1_KNOBS): + self.descriptors = dict(descriptors) + + def materialize( + self, + case: ExperimentCase, + adapter: RuntimeMaterializer, + ) -> RuntimeMaterialization: + runtime_kind = _adapter_identity(adapter, "runtime_kind") + adapter_implementation_fingerprint = _adapter_identity( + adapter, + "implementation_fingerprint", + ) + normalized = _plain_mapping(case.requested) + adapter_result = adapter.materialize(normalized, self.descriptors) + if not isinstance(adapter_result, AdapterMaterialization): + raise RuntimeMaterializationError("runtime adapter must return AdapterMaterialization") + if not isinstance(adapter_result.binding, RuntimeBinding): + raise RuntimeMaterializationError("runtime adapter must return a RuntimeBinding") + if adapter_result.binding.runtime_kind != runtime_kind: + raise RuntimeMaterializationError( + "runtime binding kind must match the materializer runtime_kind" + ) + applications = tuple(adapter_result.applications) + _validate_application_contract(normalized, applications, self.descriptors) + materialized = _mapping_from_applications(applications, "materialized") + actual = _mapping_from_applications(applications, "actual") + status = _aggregate_status(application.status for application in applications) + construction_fingerprint = _scope_fingerprint( + runtime_kind, + adapter_implementation_fingerprint, + applications, + scopes=( + IsolationScope.ENGINE_CONSTRUCTION, + IsolationScope.DISTRIBUTED_CONTEXT, + IsolationScope.PROCESS, + ), + ) + distributed_fingerprint = _scope_fingerprint( + runtime_kind, + adapter_implementation_fingerprint, + applications, + scopes=(IsolationScope.DISTRIBUTED_CONTEXT, IsolationScope.PROCESS), + ) + process_fingerprint = _scope_fingerprint( + runtime_kind, + adapter_implementation_fingerprint, + applications, + scopes=(IsolationScope.PROCESS,), + ) + isolation_scope = _strongest_scope( + [ + self.descriptors[path].lifecycle + for path in (case.changed_paths or tuple(_flatten(normalized))) + ] + ) + evidence = { + "runtime_kind": runtime_kind, + "execution_binding": case.execution_binding, + "adapter_implementation_fingerprint": adapter_implementation_fingerprint, + "binding_fingerprint": _fingerprint(adapter_result.binding.to_dict()), + "applications": { + application.path: application.to_dict() for application in applications + }, + } + materialized_case = MaterializedCase( + case=case, + normalized=normalized, + materialized=materialized, + isolation_scope=isolation_scope, + construction_fingerprint=construction_fingerprint, + distributed_context_fingerprint=distributed_fingerprint, + process_fingerprint=process_fingerprint, + status=status, + evidence=evidence, + ) + provenance = RuntimeProvenance( + requested=_plain_mapping(case.requested), + normalized=normalized, + materialized=materialized, + actual=actual, + status=status, + construction_fingerprint=construction_fingerprint, + distributed_context_fingerprint=distributed_fingerprint, + process_fingerprint=process_fingerprint, + implementation_fingerprint=adapter_implementation_fingerprint, + evidence=evidence, + ) + return RuntimeMaterialization( + materialized_case=materialized_case, + provenance=provenance, + applications=applications, + binding=adapter_result.binding, + ) + + @staticmethod + def require_executable( + materialization: RuntimeMaterialization, + *, + strict: bool, + ) -> None: + status = materialization.materialized_case.status + rejected = { + MaterializationStatus.ERROR, + MaterializationStatus.UNSUPPORTED, + MaterializationStatus.UNOBSERVABLE, + } + if strict: + rejected.add(MaterializationStatus.FALLBACK) + if status in rejected: + problems = [ + f"{application.path}={application.status.value}: " + f"{application.evidence.get('reason', 'no evidence')}" + for application in materialization.applications + if application.status is not MaterializationStatus.APPLIED + ] + raise RuntimeMaterializationError( + f"case {materialization.materialized_case.case.case_id} is not executable: " + + "; ".join(problems) + ) + + @staticmethod + def can_reuse(previous: RuntimeMaterialization, current: RuntimeMaterialization) -> bool: + """Reuse only exact semantic and implementation identities with matching state.""" + + previous_case = previous.materialized_case + current_case = current.materialized_case + return ( + previous_case.status is MaterializationStatus.APPLIED + and current_case.status is MaterializationStatus.APPLIED + and previous_case.case.identity == current_case.case.identity + and previous_case.case.execution_binding == current_case.case.execution_binding + and previous.provenance.implementation_fingerprint + == current.provenance.implementation_fingerprint + and previous_case.process_fingerprint == current_case.process_fingerprint + and previous_case.distributed_context_fingerprint + == current_case.distributed_context_fingerprint + and previous_case.construction_fingerprint == current_case.construction_fingerprint + ) + + +def _aggregate_status(statuses: Iterable[MaterializationStatus]) -> MaterializationStatus: + priority = ( + MaterializationStatus.ERROR, + MaterializationStatus.UNSUPPORTED, + MaterializationStatus.UNOBSERVABLE, + MaterializationStatus.FALLBACK, + MaterializationStatus.APPLIED, + ) + status_set = set(statuses) + if not status_set: + return MaterializationStatus.ERROR + return next(status for status in priority if status in status_set) + + +def _validate_application_contract( + normalized: Mapping[str, Any], + applications: tuple[KnobApplication, ...], + descriptors: Mapping[str, KnobDescriptor], +) -> None: + expected = _flatten(normalized) + observed_paths = [application.path for application in applications] + duplicate_paths = sorted(path for path in set(observed_paths) if observed_paths.count(path) > 1) + missing_paths = sorted(set(expected).difference(observed_paths)) + unknown_paths = sorted(set(observed_paths).difference(expected)) + problems: list[str] = [] + if missing_paths: + problems.append(f"missing paths={missing_paths!r}") + if duplicate_paths: + problems.append(f"duplicate paths={duplicate_paths!r}") + if unknown_paths: + problems.append(f"unknown paths={unknown_paths!r}") + for application in applications: + descriptor = descriptors.get(application.path) + if descriptor is None or application.path not in expected: + continue + if _plain_value(application.requested) != _plain_value(expected[application.path]): + problems.append(f"{application.path} requested value differs from normalized case") + if application.lifecycle is not descriptor.lifecycle: + problems.append(f"{application.path} lifecycle differs from descriptor") + if application.critical is not descriptor.critical: + problems.append(f"{application.path} critical flag differs from descriptor") + if problems: + raise RuntimeMaterializationError( + "runtime adapter returned invalid V1 knob applications: " + "; ".join(problems) + ) + + +def _mapping_from_applications( + applications: Sequence[KnobApplication], attribute: str +) -> dict[str, Any]: + result: dict[str, Any] = {} + for application in applications: + _set_path(result, application.path, getattr(application, attribute)) + return result + + +def _scope_fingerprint( + runtime_kind: str, + implementation_fingerprint: str, + applications: Sequence[KnobApplication], + *, + scopes: Sequence[IsolationScope], +) -> str: + scope_set = set(scopes) + values = { + application.path: application.materialized + for application in applications + if application.lifecycle in scope_set + } + return _fingerprint( + { + "runtime_kind": runtime_kind, + "implementation_fingerprint": implementation_fingerprint, + "values": values, + } + ) + + +def _strongest_scope(scopes: Sequence[IsolationScope]) -> IsolationScope: + order = { + IsolationScope.REQUEST: 0, + IsolationScope.ENGINE_CONSTRUCTION: 1, + IsolationScope.DISTRIBUTED_CONTEXT: 2, + IsolationScope.PROCESS: 3, + } + return max(scopes, key=order.__getitem__, default=IsolationScope.REQUEST) + + +def _flatten(value: Mapping[str, Any], prefix: str = "") -> dict[str, Any]: + result: dict[str, Any] = {} + for key, child in value.items(): + path = f"{prefix}.{key}" if prefix else key + if isinstance(child, Mapping): + result.update(_flatten(child, path)) + else: + result[path] = child + return result + + +def _set_path(value: dict[str, Any], path: str, child: Any) -> None: + current = value + parts = path.split(".") + for part in parts[:-1]: + current = current.setdefault(part, {}) + current[parts[-1]] = child + + +def _plain_mapping(value: Mapping[str, Any]) -> dict[str, Any]: + return {str(key): _plain_value(item) for key, item in value.items()} + + +def _plain_value(value: Any) -> Any: + if isinstance(value, Mapping): + return {str(key): _plain_value(item) for key, item in value.items()} + if isinstance(value, (tuple, list)): + return [_plain_value(item) for item in value] + if isinstance(value, (set, frozenset)): + return [_plain_value(item) for item in sorted(value, key=repr)] + return value + + +def _freeze_value(value: Any) -> Any: + if isinstance(value, Mapping): + return MappingProxyType({str(key): _freeze_value(item) for key, item in value.items()}) + if isinstance(value, (tuple, list)): + return tuple(_freeze_value(item) for item in value) + if isinstance(value, (set, frozenset)): + return frozenset(_freeze_value(item) for item in value) + return value + + +def _freeze_mapping(value: Mapping[str, Any]) -> Mapping[str, Any]: + return _freeze_value(value) + + +def _fingerprint(value: Mapping[str, Any]) -> str: + payload = json.dumps( + _plain_mapping(value), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def _adapter_identity(adapter: RuntimeMaterializer, attribute: str) -> str: + value = getattr(adapter, attribute, None) + if not isinstance(value, str) or not value.strip(): + raise RuntimeMaterializationError(f"runtime adapter {attribute} must be a non-empty string") + return value.strip() + + +__all__ = [ + "AdapterMaterialization", + "KnobApplication", + "RuntimeBinding", + "RuntimeMaterialization", + "RuntimeMaterializationError", + "RuntimeMaterializer", + "RuntimeTools", +] diff --git a/rl_engine/executors/stateless_executor.py b/rl_engine/executors/stateless_executor.py index 2047218f..70a25f99 100644 --- a/rl_engine/executors/stateless_executor.py +++ b/rl_engine/executors/stateless_executor.py @@ -16,6 +16,7 @@ StatelessForwardMode = Literal["reference", "reward", "both"] StatelessAttentionBackend = Literal["flash_attention_2", "sdpa", "eager", "model_default"] RewardAdapter = Callable[["StatelessForwardOutputs", "StatelessForwardInputs"], torch.Tensor] +SelectedLogprobCallable = Callable[..., torch.Tensor] _MISSING = object() @@ -57,6 +58,7 @@ class StatelessForwardInputs: attention_mask: torch.Tensor completion_mask: torch.Tensor labels: Optional[torch.Tensor] = None + position_ids: Optional[torch.Tensor] = None @dataclass(frozen=True) @@ -105,10 +107,12 @@ def __init__( config: Optional[StatelessForwardConfig] = None, *, reward_adapter: Optional[RewardAdapter] = None, + selected_logprob_fn: Optional[SelectedLogprobCallable] = None, ): self.model = model self.config = config or StatelessForwardConfig() self.reward_adapter = reward_adapter or default_reward_adapter + self.selected_logprob_fn = selected_logprob_fn def score(self, inputs: StatelessForwardInputs) -> StatelessForwardResult: _validate_inputs(inputs, self.config) @@ -170,6 +174,7 @@ def score(self, inputs: StatelessForwardInputs) -> StatelessForwardResult: inputs, temperature=self.config.temperature, output_dtype=self.config.output_dtype, + selected_logprob_fn=self.selected_logprob_fn, ) if self.config.return_token_scores: token_scores = reference_logps @@ -215,8 +220,14 @@ def score_reference_logprobs( *, temperature: float = 1.0, output_dtype: torch.dtype = torch.float32, + selected_logprob_fn: Optional[SelectedLogprobCallable] = None, ) -> torch.Tensor: - """Compute causal next-token selected logprobs aligned to ``[B, S]`` masks.""" + """Compute causal next-token selected logprobs aligned to ``[B, S]`` masks. + + ``selected_logprob_fn`` is an exact injection seam with the same callable + contract as :func:`selected_logprobs_reference`. Leaving it unset preserves + the historical PyTorch-reference behavior. + """ if logits.ndim != 3: raise ValueError(f"reference logits must have shape [B, S, V], got {tuple(logits.shape)}") @@ -237,13 +248,21 @@ def score_reference_logprobs( shifted_logits = logits[:, :-1, :] shifted_labels = labels[:, 1:] shifted_mask = _bool_mask(inputs.completion_mask[:, 1:], device=logits.device) - shifted_logps = selected_logprobs_reference( + scorer = selected_logprob_fn or selected_logprobs_reference + shifted_logps = scorer( shifted_logits, shifted_labels.to(device=logits.device), mask=shifted_mask, temperature=temperature, output_dtype=output_dtype, ) + if not isinstance(shifted_logps, torch.Tensor): + raise TypeError("selected_logprob_fn must return a torch.Tensor") + if shifted_logps.shape != shifted_labels.shape: + raise ValueError( + "selected_logprob_fn output shape must match selected token IDs, got " + f"{tuple(shifted_logps.shape)} and {tuple(shifted_labels.shape)}" + ) result = torch.zeros( inputs.input_ids.shape, device=logits.device, @@ -372,11 +391,20 @@ def _temporarily_configure_stateless_model( config: StatelessForwardConfig, ) -> Iterator[dict[str, float | int | str | bool]]: saved = _model_config_snapshot(model, config) - policy = configure_stateless_model(model, config) + saved_training_modes = tuple((module, module.training) for module in model.modules()) + model.eval() try: + policy = configure_stateless_model(model, config) + policy["model_eval_during_forward"] = True yield policy finally: - _restore_model_config_snapshot(saved) + try: + _restore_model_config_snapshot(saved) + finally: + # Restore each module directly. Calling ``model.train(...)`` would + # flatten intentionally mixed child-module modes. + for module, was_training in saved_training_modes: + module.training = was_training def extract_kv_cache_outputs(raw_outputs: Any) -> Optional[Any]: @@ -450,12 +478,16 @@ def _validate_inputs(inputs: StatelessForwardInputs, config: StatelessForwardCon raise ValueError("completion_mask shape must match input_ids shape") if inputs.labels is not None and inputs.labels.shape != input_ids.shape: raise ValueError("labels shape must match input_ids shape") + if inputs.position_ids is not None and inputs.position_ids.shape != input_ids.shape: + raise ValueError("position_ids shape must match input_ids shape") if attention_mask.device != input_ids.device: raise ValueError("attention_mask device must match input_ids device") if completion_mask.device != input_ids.device: raise ValueError("completion_mask device must match input_ids device") if inputs.labels is not None and inputs.labels.device != input_ids.device: raise ValueError("labels device must match input_ids device") + if inputs.position_ids is not None and inputs.position_ids.device != input_ids.device: + raise ValueError("position_ids device must match input_ids device") if config.max_batch_size is not None and input_ids.shape[0] > config.max_batch_size: raise ValueError( f"batch size {input_ids.shape[0]} exceeds max_batch_size {config.max_batch_size}" @@ -479,6 +511,10 @@ def _run_no_cache_forward( "input_ids": inputs.input_ids, "attention_mask": inputs.attention_mask, } + if inputs.position_ids is not None: + if not _call_accepts_keyword(model, "position_ids"): + raise ValueError("model does not accept the canonical batch position_ids") + kwargs["position_ids"] = inputs.position_ids if _call_accepts_keyword(model, "use_cache"): kwargs["use_cache"] = False return model(**kwargs), True From 93237f67141ccd10731d455d4fef0756cb085f55 Mon Sep 17 00:00:00 2001 From: CyberSecurityErial <2710555967@qq.com> Date: Sun, 19 Jul 2026 08:32:57 +0800 Subject: [PATCH 4/8] feat(alignment): add execution and artifact primitives --- .../alignment/cross_config/_execution.py | 617 ++++++++++++++++++ rl_engine/alignment/cross_config/artifacts.py | 473 ++++++++++++++ 2 files changed, 1090 insertions(+) create mode 100644 rl_engine/alignment/cross_config/_execution.py create mode 100644 rl_engine/alignment/cross_config/artifacts.py diff --git a/rl_engine/alignment/cross_config/_execution.py b/rl_engine/alignment/cross_config/_execution.py new file mode 100644 index 00000000..a860b951 --- /dev/null +++ b/rl_engine/alignment/cross_config/_execution.py @@ -0,0 +1,617 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Private scoring contracts and child-process supervision.""" + +from __future__ import annotations + +import hashlib +import inspect +import json +import multiprocessing as mp +import os +import tempfile +import time +import traceback +from contextlib import contextmanager +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path +from typing import Any, Iterator, Mapping, Optional, Protocol, Sequence + +import torch + +from rl_engine.alignment.cross_config._json import strict_json_loads +from rl_engine.alignment.cross_config.schema import CanonicalScoringBatch, ScorerSpec, ScoreSide + + +class PairedRunnerError(RuntimeError): + """Base error for a paired scoring attempt.""" + + +class OperatorExecutionError(PairedRunnerError): + """Raised when exact operator evidence cannot authorize execution.""" + + +class ChildScoringError(PairedRunnerError): + """Raised when a scoring child exits without a valid result.""" + + +class ScoringTimeoutError(PairedRunnerError): + """Raised after all scoring children are stopped at the deadline.""" + + +class RankCompletenessError(PairedRunnerError): + """Raised when rank results are missing, duplicated, or inconsistent.""" + + +class ScorerIdentityError(PairedRunnerError): + """Raised when paired scorer model state is not logically identical.""" + + +@dataclass(frozen=True) +class RankScore: + """One rank's full canonical selected-logprob observation.""" + + rank: int + world_size: int + selected_logprobs: torch.Tensor + metadata: Mapping[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + if self.rank < 0: + raise ValueError("rank must be >= 0") + if self.world_size < 1: + raise ValueError("world_size must be >= 1") + if self.rank >= self.world_size: + raise ValueError("rank must be less than world_size") + if not isinstance(self.selected_logprobs, torch.Tensor): + raise TypeError("selected_logprobs must be a torch.Tensor") + object.__setattr__( + self, + "selected_logprobs", + self.selected_logprobs.detach().to(device="cpu").clone(), + ) + object.__setattr__(self, "metadata", dict(self.metadata)) + + +class PairedScorer(Protocol): + """Small injection boundary used by the paired-runner control plane.""" + + spec: ScorerSpec + + def score( + self, + batch: CanonicalScoringBatch, + *, + batch_size: int, + operator: Any, + ) -> torch.Tensor | RankScore | Sequence[RankScore]: ... + + +class ChildSupervisor: + """Own the lifecycle of the two isolated scoring children.""" + + def __init__(self, start_method: Optional[str] = None): + available = mp.get_all_start_methods() + resolved = start_method or ("fork" if "fork" in available else "spawn") + if resolved not in available: + raise ValueError(f"multiprocessing start method is unavailable: {resolved}") + self.start_method = resolved + self._active_processes: list[mp.Process] = [] + + @property + def active_child_pids(self) -> tuple[int, ...]: + return tuple( + process.pid + for process in self._active_processes + if process.pid is not None and process.is_alive() + ) + + def run( + self, + attempt_dir: Path, + batch: CanonicalScoringBatch, + *, + batch_size: int, + scorers: Mapping[str, PairedScorer], + specs: Mapping[str, ScorerSpec], + instances: Mapping[str, Any], + timeout_seconds: float, + ) -> dict[str, Mapping[str, Any]]: + context: Any = mp.get_context(self.start_method) + processes: dict[str, mp.Process] = {} + with tempfile.TemporaryDirectory(prefix=".paired-runner-", dir=attempt_dir) as tmp: + temporary_dir = Path(tmp) + try: + for target in ("rollout", "training"): + process = context.Process( + target=_score_child, + name=f"cross-config-{target}", + args=( + temporary_dir / f"{target}.pt", + temporary_dir / f"{target}.error.json", + scorers[target], + specs[target], + batch, + batch_size, + instances[target], + ), + ) + process.start() + processes[target] = process + self._active_processes = list(processes.values()) + self._wait( + processes, + temporary_dir, + timeout_seconds=timeout_seconds, + ) + return { + target: _load_child_result(temporary_dir / f"{target}.pt") + for target in ("rollout", "training") + } + finally: + _stop_processes(tuple(processes.values())) + self._active_processes = [] + + @staticmethod + def _wait( + processes: Mapping[str, mp.Process], + temporary_dir: Path, + *, + timeout_seconds: float, + ) -> None: + deadline = time.monotonic() + timeout_seconds + unfinished = set(processes) + while unfinished: + for target in tuple(unfinished): + process = processes[target] + process.join(timeout=0.01) + if process.is_alive(): + continue + unfinished.remove(target) + if process.exitcode != 0: + detail = _child_error_detail(temporary_dir / f"{target}.error.json") + raise ChildScoringError( + f"{target} scoring child failed with exit code " + f"{process.exitcode}: {detail}" + ) + if unfinished and time.monotonic() >= deadline: + labels = ", ".join(sorted(unfinished)) + raise ScoringTimeoutError( + f"paired scoring exceeded {timeout_seconds:.3f}s; " + f"stopped children: {labels}" + ) + + +def _score_child( + result_path: Path, + error_path: Path, + scorer: PairedScorer, + spec: ScorerSpec, + batch: CanonicalScoringBatch, + batch_size: int, + operator: Any, +) -> None: + try: + with _read_only_scoring_guard(scorer, verify_state=True) as evidence: + with torch.no_grad(): + output = scorer.score(batch, batch_size=batch_size, operator=operator) + ranks = _coerce_rank_scores(output, spec.world_size) + payload = { + "schema_version": 1, + "guard_evidence": evidence, + "ranks": [ + { + "rank": rank.rank, + "world_size": rank.world_size, + "selected_logprobs": rank.selected_logprobs, + "metadata": json_safe(rank.metadata), + } + for rank in ranks + ], + } + temporary = result_path.with_suffix(".tmp") + torch.save(payload, temporary) + os.replace(temporary, result_path) + except BaseException as exc: + error = { + "type": f"{type(exc).__module__}.{type(exc).__qualname__}", + "message": str(exc), + "traceback": traceback.format_exc(), + } + error_path.write_text(json.dumps(error, sort_keys=True), encoding="utf-8") + raise SystemExit(1) from None + + +@contextmanager +def _read_only_scoring_guard( + scorer: PairedScorer, + *, + verify_state: bool, +) -> Iterator[dict[str, Any]]: + model = scorer_model(scorer) + if verify_state and getattr(scorer, "optimizer", None) is not None: + raise ValueError("scorer must not own an active optimizer") + if model is None: + yield { + "model_state_verified": False, + "model_eval": False, + "no_grad": True, + "optimizer_step": False, + } + return + + modes = tuple((module, module.training) for module in model.modules()) + snapshot = _module_tensor_snapshot(model) if verify_state else None + model.eval() + evidence = { + "model_state_verified": verify_state, + "model_eval": True, + "no_grad": True, + "optimizer_step": False, + "model_modes_restored": False, + "model_state_unchanged": False if verify_state else None, + } + try: + yield evidence + finally: + for module, was_training in modes: + module.training = was_training + evidence["model_modes_restored"] = True + if snapshot is not None: + mutations = _module_state_mutations(model, snapshot) + if mutations: + raise RuntimeError( + "read-only scorer mutated model parameters/buffers: " + ", ".join(mutations) + ) + evidence["model_state_unchanged"] = True + + +def _module_tensor_snapshot(model: torch.nn.Module) -> dict[str, torch.Tensor]: + values = { + f"parameter:{name}": tensor.detach().to(device="cpu").clone() + for name, tensor in model.named_parameters() + } + values.update( + { + f"buffer:{name}": tensor.detach().to(device="cpu").clone() + for name, tensor in model.named_buffers() + } + ) + return values + + +def _module_state_mutations( + model: torch.nn.Module, + before: Mapping[str, torch.Tensor], +) -> list[str]: + after = _module_tensor_snapshot(model) + mutations: list[str] = [] + for name in sorted(set(before) | set(after)): + left = before.get(name) + right = after.get(name) + if left is None or right is None: + mutations.append(name) + continue + if left.dtype != right.dtype or left.shape != right.shape or not torch.equal(left, right): + mutations.append(name) + return mutations + + +def scorer_model(scorer: PairedScorer) -> Optional[torch.nn.Module]: + if isinstance(scorer, torch.nn.Module): + return scorer + candidate = getattr(scorer, "model", None) + return candidate if isinstance(candidate, torch.nn.Module) else None + + +def paired_model_state_fingerprints( + rollout_scorer: PairedScorer, + training_scorer: PairedScorer, +) -> dict[str, Optional[str]]: + fingerprints = { + "rollout": _scorer_model_state_fingerprint(rollout_scorer), + "training": _scorer_model_state_fingerprint(training_scorer), + } + if fingerprints["rollout"] is None or fingerprints["training"] is None: + raise ScorerIdentityError( + "rollout and training model state fingerprints must both be observable" + ) + if fingerprints["rollout"] != fingerprints["training"]: + raise ScorerIdentityError( + "rollout and training model state fingerprints differ before scoring" + ) + return fingerprints + + +def _scorer_model_state_fingerprint(scorer: PairedScorer) -> Optional[str]: + declared = getattr(scorer, "model_state_fingerprint", None) + if declared is not None and (not isinstance(declared, str) or not declared): + raise ScorerIdentityError("scorer model_state_fingerprint must be a non-empty string") + model = scorer_model(scorer) + if model is None: + return declared + observed_model = _module_state_fingerprint(model) + if declared is not None and declared != observed_model: + raise ScorerIdentityError( + "declared scorer model_state_fingerprint does not match observed model state" + ) + return observed_model + + +def scorer_implementation_fingerprint(scorer: PairedScorer) -> str: + declared = getattr(scorer, "implementation_fingerprint", None) + if declared is not None and (not isinstance(declared, str) or not declared): + raise ScorerIdentityError("scorer implementation_fingerprint must be a non-empty string") + scorer_type = f"{type(scorer).__module__}.{type(scorer).__qualname__}" + score_source = _source_text(getattr(type(scorer), "score", None)) + return canonical_fingerprint( + { + "declared_implementation": declared, + "scorer_type": scorer_type, + "score_source_fingerprint": hashlib.sha256(score_source.encode("utf-8")).hexdigest(), + } + ) + + +def _module_state_fingerprint(model: torch.nn.Module) -> str: + digest = hashlib.sha256() + digest.update(f"{type(model).__module__}.{type(model).__qualname__}".encode("utf-8")) + digest.update(_source_text(type(model)).encode("utf-8")) + tensors = tuple( + (f"parameter:{name}", tensor) for name, tensor in model.named_parameters() + ) + tuple((f"buffer:{name}", tensor) for name, tensor in model.named_buffers()) + for name, tensor in tensors: + snapshot = tensor.detach().to(device="cpu") + if snapshot.is_sparse: + snapshot = snapshot.to_dense() + snapshot = snapshot.contiguous() + digest.update(name.encode("utf-8")) + digest.update(str(snapshot.dtype).encode("utf-8")) + digest.update(str(tuple(snapshot.shape)).encode("utf-8")) + digest.update(snapshot.reshape(-1).view(torch.uint8).numpy().tobytes()) + return digest.hexdigest() + + +def _source_text(value: Any) -> str: + try: + return inspect.getsource(value) + except (OSError, TypeError): + return repr(value) + + +def _coerce_rank_scores( + output: torch.Tensor | RankScore | Sequence[RankScore], + expected_world_size: int, +) -> tuple[RankScore, ...]: + if isinstance(output, torch.Tensor): + if expected_world_size != 1: + raise RankCompletenessError( + "a bare tensor result is valid only for a world_size=1 scorer" + ) + return (RankScore(rank=0, world_size=1, selected_logprobs=output),) + if isinstance(output, RankScore): + return (output,) + if not isinstance(output, Sequence) or isinstance(output, (str, bytes)): + raise TypeError("scorer must return a tensor, RankScore, or sequence of RankScore") + values = tuple(output) + if not all(isinstance(value, RankScore) for value in values): + raise TypeError("every scorer sequence item must be a RankScore") + return values + + +def _load_child_result(path: Path) -> Mapping[str, Any]: + if not path.is_file(): + raise ChildScoringError(f"scoring child produced no result artifact: {path.name}") + try: + payload = torch.load(path, map_location="cpu", weights_only=True) + except Exception as exc: + raise ChildScoringError(f"failed to load scoring child result {path.name}: {exc}") from exc + if not isinstance(payload, Mapping) or payload.get("schema_version") != 1: + raise ChildScoringError(f"malformed scoring child result: {path.name}") + return payload + + +def validate_rank_outputs( + payload: Mapping[str, Any], + spec: ScorerSpec, + *, + expected_shape: torch.Size, + target: str, +) -> dict[int, RankScore]: + raw_ranks = payload.get("ranks") + if not isinstance(raw_ranks, Sequence): + raise RankCompletenessError(f"{target} child result has no rank sequence") + ranks: dict[int, RankScore] = {} + duplicates: list[int] = [] + for raw in raw_ranks: + if not isinstance(raw, Mapping): + raise RankCompletenessError(f"{target} rank result must be a mapping") + rank_score = RankScore( + rank=int(raw["rank"]), + world_size=int(raw["world_size"]), + selected_logprobs=raw["selected_logprobs"], + metadata=raw.get("metadata", {}), + ) + if rank_score.rank in ranks: + duplicates.append(rank_score.rank) + ranks[rank_score.rank] = rank_score + if duplicates: + raise RankCompletenessError(f"{target} returned duplicate ranks: {sorted(set(duplicates))}") + expected = set(range(spec.world_size)) + actual = set(ranks) + if actual != expected: + missing = sorted(expected - actual) + unexpected = sorted(actual - expected) + raise RankCompletenessError( + f"{target} rank set is incomplete; missing={missing}, unexpected={unexpected}" + ) + for rank_index, value in ranks.items(): + if value.world_size != spec.world_size: + raise RankCompletenessError( + f"{target} rank {rank_index} reported world_size={value.world_size}, " + f"expected {spec.world_size}" + ) + if value.selected_logprobs.shape != expected_shape: + raise RankCompletenessError( + f"{target} rank {rank_index} selected_logprobs shape " + f"{tuple(value.selected_logprobs.shape)} does not match " + f"canonical shape {tuple(expected_shape)}" + ) + expected_dtype = torch_dtype(spec.dtype) + if ( + not value.selected_logprobs.is_floating_point() + or value.selected_logprobs.dtype != expected_dtype + ): + raise RankCompletenessError( + f"{target} rank {rank_index} selected_logprobs dtype " + f"{value.selected_logprobs.dtype} does not match scorer dtype {expected_dtype}" + ) + rank_zero = ranks[0].selected_logprobs + for rank_index, value in ranks.items(): + if rank_index == 0: + continue + if value.selected_logprobs.dtype != rank_zero.dtype or not torch.equal( + value.selected_logprobs, + rank_zero, + ): + raise RankCompletenessError( + f"{target} rank {rank_index} selected_logprobs diverge from rank 0" + ) + return ranks + + +def scorer_spec(scorer: PairedScorer, expected_side: ScoreSide) -> ScorerSpec: + spec = getattr(scorer, "spec", None) + if not isinstance(spec, ScorerSpec): + raise TypeError("paired scorer must expose a ScorerSpec as .spec") + if spec.side is not expected_side: + raise ValueError(f"scorer side {spec.side.value!r} does not match {expected_side.value!r}") + model = scorer_model(scorer) + if model is not None: + _require_module_on_device(model, device_type(spec.device)) + _require_module_float_dtype(model, torch_dtype(spec.dtype)) + return spec + + +def validate_scorer_identity( + specs: Mapping[str, ScorerSpec], + batch: CanonicalScoringBatch, +) -> None: + identity = batch.identity + expected = { + "checkpoint_id": identity.checkpoint_id, + "model_version": identity.model_version, + "pre_update_state": identity.pre_update_state, + } + for target in ("rollout", "training"): + observed = specs[target].construction_options + mismatches = [key for key, value in expected.items() if observed.get(key) != value] + if mismatches: + raise ScorerIdentityError( + f"{target} scorer construction identity differs from canonical identity: " + + ", ".join(mismatches) + ) + + +def _child_error_detail(path: Path) -> str: + try: + value = _read_json_object(path) + except (OSError, ValueError, json.JSONDecodeError): + return "child did not publish structured error evidence" + return f"{value.get('type', 'error')}: {value.get('message', '')}" + + +def _read_json_object(path: Path) -> dict[str, Any]: + value = strict_json_loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"expected a JSON object: {path}") + return value + + +def _stop_processes(processes: Iterator[mp.Process] | Sequence[mp.Process]) -> None: + values = tuple(processes) + for process in values: + if process.is_alive(): + process.terminate() + for process in values: + if process.pid is not None: + process.join(timeout=1.0) + for process in values: + if process.is_alive() and hasattr(process, "kill"): + process.kill() + process.join(timeout=1.0) + + +def _require_module_on_device(model: torch.nn.Module, expected: str) -> None: + for name, tensor in tuple(model.named_parameters()) + tuple(model.named_buffers()): + if tensor.device.type != expected: + raise ValueError( + f"scorer model tensor {name!r} is on {tensor.device}; expected {expected}" + ) + + +def _require_module_float_dtype(model: torch.nn.Module, expected: torch.dtype) -> None: + mismatches = [ + f"{name}={tensor.dtype}" + for name, tensor in tuple(model.named_parameters()) + tuple(model.named_buffers()) + if tensor.is_floating_point() and tensor.dtype != expected + ] + if mismatches: + raise ValueError( + f"scorer floating model state must use {expected}: " + ", ".join(mismatches) + ) + + +def device_type(value: str) -> str: + try: + return torch.device(value).type + except (TypeError, RuntimeError) as exc: + raise ValueError(f"invalid scorer device: {value!r}") from exc + + +def normalized_dtype(value: str) -> str: + dtype = torch_dtype(value) + return str(dtype).removeprefix("torch.") + + +def torch_dtype(value: str) -> torch.dtype: + normalized = str(value).strip().lower().replace("torch.", "") + dtypes = { + "float32": torch.float32, + "fp32": torch.float32, + "float16": torch.float16, + "fp16": torch.float16, + "bfloat16": torch.bfloat16, + "bf16": torch.bfloat16, + } + try: + return dtypes[normalized] + except KeyError as exc: + raise ValueError(f"unsupported stateless scorer dtype: {value!r}") from exc + + +def json_safe(value: Any) -> Any: + if isinstance(value, Enum): + return value.value + if isinstance(value, Mapping): + return {str(key): json_safe(item) for key, item in value.items()} + if isinstance(value, (set, frozenset, tuple, list)): + items = [json_safe(item) for item in value] + if isinstance(value, (set, frozenset)): + return sorted(items, key=lambda item: json.dumps(item, sort_keys=True)) + return items + if isinstance(value, (str, int, float, bool)) or value is None: + return value + return repr(value) + + +def canonical_fingerprint(value: Any) -> str: + serialized = json.dumps( + json_safe(value), + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ) + return hashlib.sha256(serialized.encode("utf-8")).hexdigest() diff --git a/rl_engine/alignment/cross_config/artifacts.py b/rl_engine/alignment/cross_config/artifacts.py new file mode 100644 index 00000000..b1b1785a --- /dev/null +++ b/rl_engine/alignment/cross_config/artifacts.py @@ -0,0 +1,473 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Append-only, crash-safe artifacts for cross-configuration runs.""" + +from __future__ import annotations + +import json +import os +import tempfile +from pathlib import Path +from typing import Any, Iterable, Mapping, Optional + +import torch + +from rl_engine.alignment.cross_config._json import strict_json_loads + +REQUIRED_CASE_ARTIFACTS = frozenset( + { + "requested.json", + "materialized.json", + "actual.json", + "identity.json", + "score_rollout.pt", + "score_training.pt", + "comparison.json", + "token_diffs.pt", + } +) +_JSON_SCHEMAS = { + "requested.json": "cross_config.requested.v1", + "materialized.json": "cross_config.materialized_envelope.v1", + "actual.json": "cross_config.actual.v1", + "identity.json": "cross_config.identity_envelope.v1", + "comparison.json": "cross_config.alignment_result.v1", +} +_JSON_REQUIRED_KEYS = { + "requested.json": frozenset({"case"}), + "materialized.json": frozenset({"materialized_case"}), + "actual.json": frozenset({"rollout", "training"}), + "identity.json": frozenset({"identity"}), + "comparison.json": frozenset({"status", "comparable", "passed"}), +} +_TENSOR_REQUIRED_KEYS = { + "score_rollout.pt": frozenset({"selected_logprobs", "active_mask"}), + "score_training.pt": frozenset({"selected_logprobs", "active_mask"}), + "token_diffs.pt": frozenset( + { + "rollout_logprobs", + "training_logprobs", + "active_mask", + "absolute_diff", + "mismatch_mask", + } + ), +} + + +class ArtifactError(RuntimeError): + """Raised when an artifact is incomplete, malformed, or would be overwritten.""" + + +class ArtifactStore: + """Persist immutable attempt directories and atomically mark completed attempts.""" + + def __init__(self, root: str | Path): + self.root = Path(root) + + def experiment_dir(self, experiment_id: str) -> Path: + return self.root / _safe_component(experiment_id, "experiment_id") + + def initialize_experiment( + self, + experiment_id: str, + *, + experiment: Mapping[str, Any], + plan: Iterable[Mapping[str, Any]], + ) -> Path: + """Create immutable experiment metadata, or verify an identical resume target.""" + + directory = self.experiment_dir(experiment_id) + directory.mkdir(parents=True, exist_ok=True) + self._write_or_verify_json(directory / "experiment.json", experiment) + plan_text = "".join(_canonical_json(item) + "\n" for item in plan) + self._write_or_verify_text(directory / "plan.jsonl", plan_text) + return directory + + def create_attempt( + self, + experiment_id: str, + case_id: str, + *, + attempt_id: Optional[str] = None, + ) -> Path: + """Allocate an append-only attempt directory for a case.""" + + case_dir = ( + self.experiment_dir(experiment_id) / "cases" / _safe_component(case_id, "case_id") + ) + case_dir.mkdir(parents=True, exist_ok=True) + if attempt_id is not None: + attempt_dir = case_dir / _safe_component(attempt_id, "attempt_id") + try: + attempt_dir.mkdir() + except FileExistsError as exc: + raise ArtifactError(f"attempt already exists: {attempt_dir}") from exc + _fsync_directory(case_dir) + return attempt_dir + + # Another controller can win after _next_attempt_id() observes the + # directory. mkdir is the atomic allocator; retry rather than aliasing or + # overwriting the winning attempt. + while True: + resolved_attempt_id = self._next_attempt_id(case_dir) + attempt_dir = case_dir / resolved_attempt_id + try: + attempt_dir.mkdir() + except FileExistsError: + continue + _fsync_directory(case_dir) + return attempt_dir + + def write_json(self, attempt_dir: str | Path, name: str, value: Mapping[str, Any]) -> Path: + path = self._attempt_path(attempt_dir, name, suffix=".json") + self._write_new_text(path, _canonical_json(value) + "\n") + return path + + def write_tensor_bundle( + self, + attempt_dir: str | Path, + name: str, + tensors: Mapping[str, torch.Tensor], + *, + metadata: Optional[Mapping[str, Any]] = None, + ) -> Path: + """Write CPU tensor payloads that can be loaded with ``weights_only=True``.""" + + path = self._attempt_path(attempt_dir, name, suffix=".pt") + payload: dict[str, Any] = { + "schema_version": 1, + "tensors": { + key: tensor.detach().to(device="cpu").contiguous() + for key, tensor in tensors.items() + }, + "metadata": dict(metadata or {}), + } + self._atomic_torch_save(path, payload) + return path + + def load_tensor_bundle(self, path: str | Path) -> dict[str, Any]: + try: + payload = torch.load(Path(path), map_location="cpu", weights_only=True) + except Exception as exc: + raise ArtifactError(f"failed to load tensor artifact {path}: {exc}") from exc + if not isinstance(payload, dict) or payload.get("schema_version") != 1: + raise ArtifactError(f"unsupported tensor artifact schema: {path}") + tensors = payload.get("tensors") + if not isinstance(tensors, dict) or not all( + isinstance(value, torch.Tensor) for value in tensors.values() + ): + raise ArtifactError(f"malformed tensor payload: {path}") + return payload + + def complete_attempt( + self, + attempt_dir: str | Path, + *, + summary: Mapping[str, Any], + required: Iterable[str] = REQUIRED_CASE_ARTIFACTS, + ) -> Path: + """Validate all payloads before publishing an atomic ``COMPLETE`` marker.""" + + directory = Path(attempt_dir) + missing = sorted(name for name in required if not (directory / name).is_file()) + if missing: + raise ArtifactError(f"cannot complete {directory}; missing artifacts: {missing}") + self._validate_machine_artifacts(directory) + self._validate_complete_summary(directory, summary) + marker = directory / "COMPLETE" + self._write_new_text(marker, _canonical_json(summary) + "\n") + return marker + + def completed_attempt( + self, + experiment_id: str, + case_id: str, + *, + required: Iterable[str] = REQUIRED_CASE_ARTIFACTS, + ) -> Optional[Path]: + """Return the newest valid completed attempt, ignoring partial attempts.""" + + case_dir = ( + self.experiment_dir(experiment_id) / "cases" / _safe_component(case_id, "case_id") + ) + if not case_dir.is_dir(): + return None + for attempt_dir in sorted( + case_dir.iterdir(), + key=_attempt_sort_key, + reverse=True, + ): + if not attempt_dir.is_dir() or not (attempt_dir / "COMPLETE").is_file(): + continue + try: + self.validate_completed_attempt( + attempt_dir, + required=required, + expected_case_id=case_id, + ) + except ArtifactError: + continue + return attempt_dir + return None + + def validate_completed_attempt( + self, + attempt_dir: str | Path, + *, + required: Iterable[str] = REQUIRED_CASE_ARTIFACTS, + expected_case_id: Optional[str] = None, + ) -> None: + directory = Path(attempt_dir) + marker = directory / "COMPLETE" + if not marker.is_file(): + raise ArtifactError(f"missing COMPLETE marker: {directory}") + try: + marker_value = strict_json_loads(marker.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError, ValueError) as exc: + raise ArtifactError(f"malformed COMPLETE marker: {marker}") from exc + if not isinstance(marker_value, dict): + raise ArtifactError(f"COMPLETE marker must contain a JSON object: {marker}") + self._validate_complete_summary( + directory, + marker_value, + expected_case_id=expected_case_id, + ) + missing = sorted(name for name in required if not (directory / name).is_file()) + if missing: + raise ArtifactError(f"completed attempt is missing artifacts: {missing}") + self._validate_machine_artifacts(directory, expected_case_id=expected_case_id) + + def _validate_machine_artifacts( + self, + directory: Path, + *, + expected_case_id: Optional[str] = None, + ) -> None: + tensor_payloads: dict[str, dict[str, Any]] = {} + for name in ("score_rollout.pt", "score_training.pt", "token_diffs.pt"): + path = directory / name + if path.exists(): + payload = self.load_tensor_bundle(path) + tensor_payloads[name] = payload + tensors = payload["tensors"] + missing_tensor_keys = sorted(_TENSOR_REQUIRED_KEYS[name].difference(tensors)) + if missing_tensor_keys: + raise ArtifactError( + f"tensor artifact {name} is missing keys: {missing_tensor_keys}" + ) + metadata = payload.get("metadata", {}) + if not isinstance(metadata, Mapping): + raise ArtifactError(f"tensor artifact metadata must be an object: {path}") + if expected_case_id is not None and metadata.get("case_id") != expected_case_id: + raise ArtifactError( + f"tensor artifact case_id does not match {expected_case_id!r}: {path}" + ) + if metadata.get("attempt_id") != directory.name: + raise ArtifactError( + f"tensor artifact attempt_id does not match {directory.name!r}: {path}" + ) + json_payloads: dict[str, dict[str, Any]] = {} + for name in ( + "requested.json", + "materialized.json", + "actual.json", + "identity.json", + "comparison.json", + ): + path = directory / name + if not path.exists(): + continue + try: + value = strict_json_loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError, ValueError) as exc: + raise ArtifactError(f"malformed JSON artifact: {path}") from exc + if not isinstance(value, dict): + raise ArtifactError(f"JSON artifact must contain an object: {path}") + json_payloads[name] = value + if expected_case_id is not None and value.get("case_id") != expected_case_id: + raise ArtifactError( + f"JSON artifact case_id does not match {expected_case_id!r}: {path}" + ) + if value.get("schema_version") != _JSON_SCHEMAS[name]: + raise ArtifactError(f"JSON artifact has an unsupported schema: {path}") + missing_json_keys = sorted(_JSON_REQUIRED_KEYS[name].difference(value)) + if missing_json_keys: + raise ArtifactError(f"JSON artifact {name} is missing keys: {missing_json_keys}") + if value.get("attempt_id") != directory.name: + raise ArtifactError( + f"JSON artifact attempt_id does not match {directory.name!r}: {path}" + ) + + if expected_case_id is None and json_payloads: + case_ids = {payload.get("case_id") for payload in json_payloads.values()} + if len(case_ids) != 1 or None in case_ids: + raise ArtifactError("JSON artifacts must declare one consistent case_id") + inferred_case_id = next(iter(case_ids)) + for name, payload in tensor_payloads.items(): + if payload["metadata"].get("case_id") != inferred_case_id: + raise ArtifactError( + f"tensor artifact case_id does not match {inferred_case_id!r}: " + f"{directory / name}" + ) + + @staticmethod + def _validate_complete_summary( + directory: Path, + summary: Mapping[str, Any], + *, + expected_case_id: Optional[str] = None, + ) -> None: + if summary.get("schema_version") != "cross_config.complete.v1": + raise ArtifactError(f"COMPLETE marker has an unsupported schema: {directory}") + case_id = summary.get("case_id") + if not isinstance(case_id, str) or not case_id: + raise ArtifactError(f"COMPLETE marker is missing case_id: {directory}") + if expected_case_id is not None and case_id != expected_case_id: + raise ArtifactError( + f"COMPLETE marker case_id does not match {expected_case_id!r}: {directory}" + ) + if summary.get("attempt_id") != directory.name: + raise ArtifactError( + f"COMPLETE marker attempt_id does not match {directory.name!r}: {directory}" + ) + if not isinstance(summary.get("status"), str): + raise ArtifactError(f"COMPLETE marker is missing status: {directory}") + + def _write_or_verify_json(self, path: Path, value: Mapping[str, Any]) -> None: + self._write_or_verify_text(path, _canonical_json(value) + "\n") + + def _write_or_verify_text(self, path: Path, text: str) -> None: + if path.exists(): + self._verify_existing_text(path, text) + return + try: + self._atomic_write_text(path, text) + except ArtifactError: + # A concurrent writer may have atomically published the same immutable + # experiment metadata. Accept only byte-identical content. + if not path.exists(): + raise + self._verify_existing_text(path, text) + + @staticmethod + def _verify_existing_text(path: Path, text: str) -> None: + try: + existing = path.read_text(encoding="utf-8") + except OSError as exc: + raise ArtifactError(f"failed to read existing artifact {path}: {exc}") from exc + if existing != text: + raise ArtifactError(f"resume metadata differs from existing artifact: {path}") + + def _write_new_text(self, path: Path, text: str) -> None: + if path.exists(): + raise ArtifactError(f"refusing to overwrite artifact: {path}") + self._atomic_write_text(path, text) + + def _atomic_write_text(self, path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fd, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + temporary = Path(temporary_name) + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(text) + handle.flush() + os.fsync(handle.fileno()) + _publish_new_file(temporary, path) + except Exception: + temporary.unlink(missing_ok=True) + raise + + def _atomic_torch_save(self, path: Path, payload: Mapping[str, Any]) -> None: + if path.exists(): + raise ArtifactError(f"refusing to overwrite artifact: {path}") + path.parent.mkdir(parents=True, exist_ok=True) + fd, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + os.close(fd) + temporary = Path(temporary_name) + try: + torch.save(dict(payload), temporary) + with temporary.open("rb") as handle: + os.fsync(handle.fileno()) + _publish_new_file(temporary, path) + except Exception: + temporary.unlink(missing_ok=True) + raise + + @staticmethod + def _next_attempt_id(case_dir: Path) -> str: + indices: list[int] = [] + for child in case_dir.iterdir(): + if not child.is_dir() or not child.name.startswith("attempt-"): + continue + suffix = child.name.removeprefix("attempt-") + if suffix.isdigit(): + indices.append(int(suffix)) + return f"attempt-{max(indices, default=0) + 1:04d}" + + @staticmethod + def _attempt_path(attempt_dir: str | Path, name: str, *, suffix: str) -> Path: + directory = Path(attempt_dir) + safe_name = _safe_component(name, "artifact name") + if not safe_name.endswith(suffix): + safe_name += suffix + return directory / safe_name + + +def _canonical_json(value: Mapping[str, Any]) -> str: + try: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ) + except (TypeError, ValueError) as exc: + raise ArtifactError(f"artifact is not strict JSON: {exc}") from exc + + +def _safe_component(value: str, label: str) -> str: + if not value or value in {".", ".."} or Path(value).name != value: + raise ValueError(f"{label} must be a single non-empty path component") + return value + + +def _attempt_sort_key(path: Path) -> tuple[int, int, str]: + """Order standard attempt IDs numerically and retain nonstandard fallbacks.""" + + prefix = "attempt-" + suffix = path.name.removeprefix(prefix) + if path.name.startswith(prefix) and suffix.isdigit(): + return (1, int(suffix), path.name) + return (0, -1, path.name) + + +def _publish_new_file(temporary: Path, destination: Path) -> None: + """Atomically publish without ever replacing an existing artifact.""" + + try: + os.link(temporary, destination) + except FileExistsError as exc: + raise ArtifactError(f"refusing to overwrite artifact: {destination}") from exc + temporary.unlink() + _fsync_directory(destination.parent) + + +def _fsync_directory(directory: Path) -> None: + """Persist directory entry changes where the host filesystem supports it.""" + + try: + descriptor = os.open(directory, os.O_RDONLY) + except OSError: + return + try: + os.fsync(descriptor) + except OSError: + pass + finally: + os.close(descriptor) + + +__all__ = ["ArtifactError", "ArtifactStore", "REQUIRED_CASE_ARTIFACTS"] From 5e83c94823ebd81bfab1a86b23aaf34ce1caa35b Mon Sep 17 00:00:00 2001 From: CyberSecurityErial <2710555967@qq.com> Date: Sun, 19 Jul 2026 08:33:36 +0800 Subject: [PATCH 5/8] feat(alignment): add provenance-aware paired execution --- rl_engine/alignment/cross_config/__init__.py | 40 + .../alignment/cross_config/_provenance.py | 327 ++++++++ rl_engine/alignment/cross_config/_resume.py | 344 +++++++++ rl_engine/alignment/cross_config/runner.py | 713 ++++++++++++++++++ 4 files changed, 1424 insertions(+) create mode 100644 rl_engine/alignment/cross_config/__init__.py create mode 100644 rl_engine/alignment/cross_config/_provenance.py create mode 100644 rl_engine/alignment/cross_config/_resume.py create mode 100644 rl_engine/alignment/cross_config/runner.py diff --git a/rl_engine/alignment/cross_config/__init__.py b/rl_engine/alignment/cross_config/__init__.py new file mode 100644 index 00000000..9eed6341 --- /dev/null +++ b/rl_engine/alignment/cross_config/__init__.py @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Plan and run cross-configuration alignment experiments. + +The package root is intentionally small and lazily loads execution code. Extension +authors import adapter, artifact, operator, or schema details from their owning +submodule. +""" + +from importlib import import_module +from typing import Any + +from rl_engine.alignment.cross_config.comparison import compare_score_artifacts +from rl_engine.alignment.cross_config.config import ExperimentConfig, load_config +from rl_engine.alignment.cross_config.execution_plan import ExecutionPlan, build_execution_plan +from rl_engine.alignment.cross_config.planner import ExperimentPlan, Planner + + +def __getattr__(name: str) -> Any: + if name in {"PairedRunResult", "PairedRunner"}: + return getattr(import_module("rl_engine.alignment.cross_config.runner"), name) + if name in {"RuntimeMaterializer", "RuntimeTools"}: + return getattr(import_module("rl_engine.alignment.cross_config.runtime"), name) + raise AttributeError(name) + + +__all__ = [ + "ExperimentConfig", + "ExperimentPlan", + "ExecutionPlan", + "PairedRunResult", + "PairedRunner", + "Planner", + "RuntimeMaterializer", + "RuntimeTools", + "compare_score_artifacts", + "build_execution_plan", + "load_config", +] diff --git a/rl_engine/alignment/cross_config/_provenance.py b/rl_engine/alignment/cross_config/_provenance.py new file mode 100644 index 00000000..e9a8d962 --- /dev/null +++ b/rl_engine/alignment/cross_config/_provenance.py @@ -0,0 +1,327 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Private execution identity and provenance construction.""" + +from __future__ import annotations + +import hashlib +import importlib.metadata +import json +import platform +from dataclasses import replace +from pathlib import Path +from typing import Any, Mapping, Optional + +import torch + +from rl_engine.alignment.cross_config._execution import ( + OperatorExecutionError, + PairedRunnerError, + canonical_fingerprint, + device_type, + json_safe, +) +from rl_engine.alignment.cross_config.runtime import RuntimeMaterialization +from rl_engine.alignment.cross_config.schema import ( + MaterializationStatus, + RuntimeProvenance, + ScoreArtifact, + ScorerSpec, + ScoreSide, +) +from rl_engine.kernels.semantic_registry import OperatorInstanceProvenance, OperatorResolution + +PAIRED_RUNNER_IMPLEMENTATION_FINGERPRINT = "cross_config.paired_runner.v1" + + +def effective_runtime_status( + materialization: RuntimeMaterialization, +) -> MaterializationStatus: + """Aggregate runtime status after exact resolution supersedes logp readback.""" + + statuses = [ + application.status + for application in materialization.applications + if not ( + application.path == "logp.backend" + and application.status is MaterializationStatus.UNOBSERVABLE + ) + ] + precedence = ( + MaterializationStatus.ERROR, + MaterializationStatus.UNSUPPORTED, + MaterializationStatus.FALLBACK, + MaterializationStatus.UNOBSERVABLE, + MaterializationStatus.APPLIED, + ) + return next( + (status for status in precedence if status in statuses), + MaterializationStatus.APPLIED, + ) + + +def side_provenance( + base: RuntimeProvenance, + resolution: OperatorResolution, + instance: OperatorInstanceProvenance, + child_payload: Mapping[str, Any], + spec: ScorerSpec, + *, + status: MaterializationStatus, + factory_options: Mapping[str, Any], + model_state_fingerprint: Optional[str], + scorer_implementation_fingerprint: str, +) -> RuntimeProvenance: + payload = base.to_dict() + actual = dict(payload["actual"]) + actual["operators"] = { + "selected_logprob": { + "backend_id": instance.backend_id, + "descriptor_fingerprint": instance.descriptor_fingerprint, + "implementation_fingerprint": instance.implementation_fingerprint, + "instance_fingerprint": instance.instance_fingerprint, + "concrete_implementation": instance.concrete_implementation, + "factory_options": json_safe(factory_options), + "factory_options_fingerprint": factory_options_fingerprint(factory_options), + } + } + actual["model_state_fingerprint"] = model_state_fingerprint + actual["scorer_implementation_fingerprint"] = scorer_implementation_fingerprint + evidence = dict(payload["evidence"]) + evidence.update( + { + "operator_resolution": resolution.to_dict(), + "operator_instance": instance.to_dict(), + "operator_factory_options": json_safe(factory_options), + "scoring_guard": json_safe(child_payload.get("guard_evidence", {})), + "rank_metadata": [ + json_safe(rank.get("metadata", {})) + for rank in child_payload.get("ranks", ()) + if isinstance(rank, Mapping) + ], + "model_state_fingerprint": model_state_fingerprint, + "scorer_implementation_fingerprint": scorer_implementation_fingerprint, + } + ) + implementation_fingerprint = hashlib.sha256( + f"{base.implementation_fingerprint}:{instance.instance_fingerprint}".encode("utf-8") + ).hexdigest() + return RuntimeProvenance( + requested=payload["requested"], + normalized=payload["normalized"], + materialized=payload["materialized"], + actual=actual, + status=status, + construction_fingerprint=base.construction_fingerprint, + distributed_context_fingerprint=base.distributed_context_fingerprint, + process_fingerprint=base.process_fingerprint, + implementation_fingerprint=implementation_fingerprint, + evidence=evidence, + rank=0, + world_size=spec.world_size, + ) + + +def concrete_scorer_spec( + spec: ScorerSpec, + instance: OperatorInstanceProvenance, +) -> ScorerSpec: + overrides = dict(spec.operator_overrides) + overrides["selected_logprob"] = instance.backend_id + return replace(spec, operator_overrides=overrides) + + +def score_metadata(artifact: ScoreArtifact) -> dict[str, Any]: + value = artifact.to_dict() + value.pop("selected_logprobs", None) + value.pop("active_mask", None) + return { + "case_id": artifact.case_id, + "attempt_id": artifact.attempt_id, + "side": artifact.side.value, + "score_artifact": value, + } + + +def execution_fingerprint( + materialization: RuntimeMaterialization, + *, + specs: Mapping[str, ScorerSpec], + instance_provenance: Mapping[str, OperatorInstanceProvenance], + operator_factory_options: Optional[Mapping[str | ScoreSide, Mapping[str, Any]]], + model_state_fingerprints: Mapping[str, Optional[str]], + scorer_implementation_fingerprints: Mapping[str, str], + environment: Mapping[str, Any], +) -> str: + payload = { + "schema_version": "cross_config.execution_identity.v1", + "runner_implementation_fingerprint": PAIRED_RUNNER_IMPLEMENTATION_FINGERPRINT, + "environment": environment, + "materialized_case": materialization.materialized_case.to_dict(), + "runtime_provenance": materialization.provenance.to_dict(), + "runtime_binding": materialization.binding.to_dict(), + "applications": [application.to_dict() for application in materialization.applications], + "targets": { + target: { + "scorer": concrete_scorer_spec( + specs[target], + instance_provenance[target], + ).to_dict(), + "operator_instance": instance_provenance[target].to_dict(), + "operator_factory_options": json_safe( + target_factory_options(operator_factory_options, target) + ), + "model_state_fingerprint": model_state_fingerprints[target], + "scorer_implementation_fingerprint": (scorer_implementation_fingerprints[target]), + } + for target in ("rollout", "training") + }, + } + return canonical_fingerprint(payload) + + +def mapping_target(mapping: Mapping[str | ScoreSide, Any], target: str) -> Any: + if target in mapping: + return mapping[target] + side = ScoreSide(target) + return mapping.get(side) + + +def target_factory_options( + options: Optional[Mapping[str | ScoreSide, Mapping[str, Any]]], + target: str, +) -> Mapping[str, Any]: + if options is None: + return {} + value = mapping_target(options, target) + if value is None: + return {} + if not isinstance(value, Mapping): + raise OperatorExecutionError(f"{target} operator factory options must be a mapping") + return dict(value) + + +def factory_options_fingerprint(options: Mapping[str, Any]) -> str: + return canonical_fingerprint(options) + + +def runtime_adapter_fingerprint(materialization: RuntimeMaterialization) -> str: + observed = materialization.provenance.evidence.get("adapter_implementation_fingerprint") + if isinstance(observed, str) and observed: + return observed + return materialization.provenance.implementation_fingerprint + + +def execution_environment_provenance( + specs: Mapping[str, ScorerSpec], + *, + runtime_adapter_fingerprint: str, + operator_implementation_fingerprints: Mapping[str, str], +) -> dict[str, Any]: + source_root = Path(__file__).resolve().parents[3] + try: + package_version = importlib.metadata.version("rl-kernel") + except importlib.metadata.PackageNotFoundError: + package_version = None + torch_config = torch.__config__.show() + execution_devices = {target: device_type(spec.device) for target, spec in sorted(specs.items())} + return { + "schema_version": "cross_config.environment.v1", + "execution_devices": execution_devices, + "python": { + "implementation": platform.python_implementation(), + "version": platform.python_version(), + }, + "torch": { + "version": str(torch.__version__), + "git_version": getattr(torch.version, "git_version", None), + "cuda_build": getattr(torch.version, "cuda", None), + "hip_build": getattr(torch.version, "hip", None), + "debug_build": bool(getattr(torch.version, "debug", False)), + "config_fingerprint": hashlib.sha256(torch_config.encode("utf-8")).hexdigest(), + }, + "host_runtime": { + "platform": platform.platform(), + "machine": platform.machine(), + "processor": platform.processor(), + "mkldnn_available": bool(torch.backends.mkldnn.is_available()), + "mkl_available": bool(torch.backends.mkl.is_available()), + }, + "rl_kernel": { + "package_version": package_version, + "git_revision": _git_revision(source_root), + "source_tree_fingerprint": _cross_config_source_tree_fingerprint( + source_root, + implementation_fingerprints={ + "runtime_adapter": runtime_adapter_fingerprint, + "operators": dict(operator_implementation_fingerprints), + }, + ), + }, + } + + +def _git_revision(source_root: Path) -> Optional[str]: + git_dir = source_root / ".git" + try: + if git_dir.is_file(): + marker = git_dir.read_text(encoding="utf-8").strip() + if not marker.startswith("gitdir: "): + return None + resolved = Path(marker.removeprefix("gitdir: ")) + git_dir = resolved if resolved.is_absolute() else source_root / resolved + head = (git_dir / "HEAD").read_text(encoding="utf-8").strip() + if not head.startswith("ref: "): + return head or None + reference = head.removeprefix("ref: ") + loose_ref = git_dir / reference + if loose_ref.is_file(): + return loose_ref.read_text(encoding="utf-8").strip() or None + packed_refs = git_dir / "packed-refs" + if packed_refs.is_file(): + suffix = f" {reference}" + for line in packed_refs.read_text(encoding="utf-8").splitlines(): + if line.endswith(suffix): + return line.split(" ", 1)[0] + except OSError: + return None + return None + + +def _cross_config_source_tree_fingerprint( + source_root: Path, + *, + implementation_fingerprints: Mapping[str, Any], +) -> str: + paths = list((source_root / "rl_engine/alignment/cross_config").glob("*.py")) + paths.extend( + source_root / relative + for relative in ( + "rl_engine/executors/stateless_executor.py", + "rl_engine/kernels/gtest/tolerance.py", + "rl_engine/kernels/registry.py", + "rl_engine/kernels/semantic_registry.py", + ) + ) + digest = hashlib.sha256() + for path in sorted(set(paths)): + try: + content = path.read_bytes() + except OSError as exc: + raise PairedRunnerError( + f"cannot fingerprint cross-configuration source file {path}: {exc}" + ) from exc + digest.update(str(path.relative_to(source_root)).encode("utf-8")) + digest.update(b"\0") + digest.update(content) + digest.update(b"\0") + digest.update( + json.dumps( + json_safe(implementation_fingerprints), + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + ) + return digest.hexdigest() diff --git a/rl_engine/alignment/cross_config/_resume.py b/rl_engine/alignment/cross_config/_resume.py new file mode 100644 index 00000000..b963fef1 --- /dev/null +++ b/rl_engine/alignment/cross_config/_resume.py @@ -0,0 +1,344 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Private validation for append-only attempt resume.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any, Mapping, Optional + +import torch + +from rl_engine.alignment.cross_config._execution import ( + canonical_fingerprint, + json_safe, + torch_dtype, +) +from rl_engine.alignment.cross_config._json import strict_json_loads +from rl_engine.alignment.cross_config._provenance import ( + PAIRED_RUNNER_IMPLEMENTATION_FINGERPRINT, + concrete_scorer_spec, + effective_runtime_status, + factory_options_fingerprint, + target_factory_options, +) +from rl_engine.alignment.cross_config.comparison import recompute_mismatch_mask +from rl_engine.alignment.cross_config.runtime import RuntimeMaterialization +from rl_engine.alignment.cross_config.schema import ( + CanonicalScoringBatch, + ExperimentCase, + ScorerSpec, + ScoreSide, +) +from rl_engine.kernels.gtest.tolerance import ( + resolve_logprob_threshold, + tolerance_contract_fingerprint, +) +from rl_engine.kernels.semantic_registry import OperatorInstanceProvenance + + +def completed_attempt_matches( + attempt_dir: Path, + case: ExperimentCase, + batch: CanonicalScoringBatch, + *, + materialization: RuntimeMaterialization, + specs: Mapping[str, ScorerSpec], + instance_provenance: Mapping[str, OperatorInstanceProvenance], + operator_factory_options: Optional[Mapping[str | ScoreSide, Mapping[str, Any]]], + model_state_fingerprints: Mapping[str, Optional[str]], + scorer_implementation_fingerprints: Mapping[str, str], + environment: Mapping[str, Any], + execution_fingerprint: str, +) -> bool: + try: + identity = read_json_object(attempt_dir / "identity.json") + requested = read_json_object(attempt_dir / "requested.json") + actual = read_json_object(attempt_dir / "actual.json") + marker = read_json_object(attempt_dir / "COMPLETE") + except (OSError, ValueError, json.JSONDecodeError): + return False + if not ( + identity.get("schema_version") == "cross_config.identity_envelope.v1" + and requested.get("schema_version") == "cross_config.requested.v1" + and actual.get("schema_version") == "cross_config.actual.v1" + and marker.get("schema_version") == "cross_config.complete.v1" + and actual.get("runner_implementation_fingerprint") + == PAIRED_RUNNER_IMPLEMENTATION_FINGERPRINT + and actual.get("environment") == environment + and actual.get("environment_fingerprint") == canonical_fingerprint(environment) + and marker.get("runner_implementation_fingerprint") + == PAIRED_RUNNER_IMPLEMENTATION_FINGERPRINT + ): + return False + if not ( + identity.get("case_id") == case.case_id + and identity.get("identity") == batch.identity.to_dict() + and requested.get("case") == case.to_dict() + and marker.get("execution_fingerprint") == execution_fingerprint + and actual.get("execution_fingerprint") == execution_fingerprint + and marker.get("rollout_backend") == instance_provenance["rollout"].backend_id + and marker.get("training_backend") == instance_provenance["training"].backend_id + ): + return False + + runtime = materialization.provenance.to_dict() + effective_status = effective_runtime_status(materialization).value + score_tensors: dict[str, Mapping[str, torch.Tensor]] = {} + for target in ("rollout", "training"): + prior = actual.get(target) + if not isinstance(prior, Mapping): + return False + instance = instance_provenance[target] + options = target_factory_options(operator_factory_options, target) + expected_operator = { + "backend_id": instance.backend_id, + "descriptor_fingerprint": instance.descriptor_fingerprint, + "implementation_fingerprint": instance.implementation_fingerprint, + "instance_fingerprint": instance.instance_fingerprint, + "concrete_implementation": instance.concrete_implementation, + "factory_options": json_safe(options), + "factory_options_fingerprint": factory_options_fingerprint(options), + } + prior_actual = prior.get("actual") + if not isinstance(prior_actual, Mapping): + return False + if any(prior_actual.get(key) != value for key, value in runtime["actual"].items()): + return False + if prior_actual.get("operators", {}).get("selected_logprob") != expected_operator: + return False + if prior_actual.get("model_state_fingerprint") != model_state_fingerprints[target]: + return False + if ( + prior_actual.get("scorer_implementation_fingerprint") + != scorer_implementation_fingerprints[target] + ): + return False + expected_implementation = hashlib.sha256( + ( + f"{materialization.provenance.implementation_fingerprint}:" + f"{instance.instance_fingerprint}" + ).encode("utf-8") + ).hexdigest() + for key, expected in ( + ("requested", runtime["requested"]), + ("normalized", runtime["normalized"]), + ("materialized", runtime["materialized"]), + ("status", effective_status), + ( + "construction_fingerprint", + materialization.provenance.construction_fingerprint, + ), + ( + "distributed_context_fingerprint", + materialization.provenance.distributed_context_fingerprint, + ), + ("process_fingerprint", materialization.provenance.process_fingerprint), + ("implementation_fingerprint", expected_implementation), + ("world_size", specs[target].world_size), + ): + if prior.get(key) != expected: + return False + try: + score_payload = _load_resume_tensor_bundle(attempt_dir / f"score_{target}.pt") + score_artifact = score_payload["metadata"]["score_artifact"] + prior_scorer = score_artifact["scorer"] + except (OSError, KeyError, TypeError, RuntimeError, ValueError): + return False + expected_scorer = concrete_scorer_spec(specs[target], instance).to_dict() + if prior_scorer != expected_scorer: + return False + if ( + score_artifact.get("schema_version") != "cross_config.score_artifact.v1" + or score_artifact.get("case_id") != case.case_id + or score_artifact.get("attempt_id") != attempt_dir.name + or score_artifact.get("side") != target + or score_artifact.get("identity") != batch.identity.to_dict() + or score_artifact.get("provenance") != prior + ): + return False + tensors = score_payload["tensors"] + selected = tensors.get("selected_logprobs") + active_mask = tensors.get("active_mask") + expected_dtype = torch_dtype(specs[target].dtype) + if ( + not isinstance(selected, torch.Tensor) + or not isinstance(active_mask, torch.Tensor) + or selected.shape != batch.input_ids.shape + or active_mask.shape != batch.input_ids.shape + or selected.dtype != expected_dtype + or active_mask.dtype != torch.bool + or not torch.equal(active_mask, batch.active_mask.to(device="cpu")) + ): + return False + metadata = score_payload["metadata"] + if ( + metadata.get("case_id") != case.case_id + or metadata.get("attempt_id") != attempt_dir.name + or metadata.get("side") != target + ): + return False + score_tensors[target] = tensors + return _resume_comparison_matches( + attempt_dir, + case, + batch, + marker, + score_tensors, + specs, + ) + + +def _load_resume_tensor_bundle(path: Path) -> Mapping[str, Any]: + payload = torch.load(path, map_location="cpu", weights_only=True) + if not isinstance(payload, Mapping) or payload.get("schema_version") != 1: + raise ValueError(f"invalid tensor bundle schema: {path}") + tensors = payload.get("tensors") + metadata = payload.get("metadata") + if not isinstance(tensors, Mapping) or not all( + isinstance(tensor, torch.Tensor) for tensor in tensors.values() + ): + raise ValueError(f"invalid tensor bundle payload: {path}") + if not isinstance(metadata, Mapping): + raise ValueError(f"invalid tensor bundle metadata: {path}") + return payload + + +def _resume_comparison_matches( + attempt_dir: Path, + case: ExperimentCase, + batch: CanonicalScoringBatch, + marker: Mapping[str, Any], + scores: Mapping[str, Mapping[str, torch.Tensor]], + specs: Mapping[str, ScorerSpec], +) -> bool: + try: + comparison = read_json_object(attempt_dir / "comparison.json") + token_bundle = _load_resume_tensor_bundle(attempt_dir / "token_diffs.pt") + except (OSError, ValueError, RuntimeError, json.JSONDecodeError): + return False + required_token_keys = { + "rollout_logprobs", + "training_logprobs", + "active_mask", + "absolute_diff", + "mismatch_mask", + } + token_tensors = token_bundle["tensors"] + if not required_token_keys.issubset(token_tensors): + return False + rollout = scores["rollout"]["selected_logprobs"] + training = scores["training"]["selected_logprobs"] + active_mask = batch.active_mask.to(device="cpu", dtype=torch.bool) + if not bool(torch.isfinite(rollout[active_mask]).all().item()) or not bool( + torch.isfinite(training[active_mask]).all().item() + ): + return False + rollout_threshold = resolve_logprob_threshold(specs["rollout"].dtype) + training_threshold = resolve_logprob_threshold(specs["training"].dtype) + if rollout_threshold != training_threshold: + return False + fixed_threshold = rollout_threshold + absolute_diff = torch.abs(training - rollout) + mismatch_mask = recompute_mismatch_mask( + rollout, + training, + active_mask, + fixed_threshold, + ) + expected_tensors = { + "rollout_logprobs": rollout, + "training_logprobs": training, + "active_mask": active_mask, + "absolute_diff": absolute_diff, + "mismatch_mask": mismatch_mask, + } + if any( + token_tensors[name].dtype != expected.dtype + or token_tensors[name].shape != expected.shape + or not torch.equal(token_tensors[name], expected) + for name, expected in expected_tensors.items() + ): + return False + token_metadata = token_bundle["metadata"] + active_count = int(active_mask.sum().item()) + if active_count == 0: + return False + mismatch_count = int(mismatch_mask.sum().item()) + passed = mismatch_count == 0 + status = "pass" if passed else "fail" + if ( + token_metadata.get("case_id") != case.case_id + or token_metadata.get("attempt_id") != attempt_dir.name + or token_metadata.get("status") != status + or token_metadata.get("fixed_threshold") != fixed_threshold + ): + return False + if ( + comparison.get("schema_version") != "cross_config.alignment_result.v1" + or comparison.get("case_id") != case.case_id + or comparison.get("attempt_id") != attempt_dir.name + or comparison.get("status") != status + or comparison.get("comparable") is not True + or comparison.get("passed") is not passed + or comparison.get("active_token_count") != active_count + or comparison.get("mismatch_count") != mismatch_count + or comparison.get("fixed_threshold") != fixed_threshold + or comparison.get("contract_fingerprint") != tolerance_contract_fingerprint() + ): + return False + serialized_token_artifact = comparison.get("token_artifact") + if not isinstance(serialized_token_artifact, Mapping): + return False + if ( + serialized_token_artifact.get("schema_version") != "cross_config.token_comparison.v1" + or serialized_token_artifact.get("fixed_threshold") != fixed_threshold + ): + return False + if any( + not _serialized_tensor_matches(serialized_token_artifact.get(name), tensor) + for name, tensor in expected_tensors.items() + ): + return False + active_diff = absolute_diff[active_mask] + max_abs_diff = float(active_diff.max().item()) + worst_flat_index = int(torch.argmax(active_diff).item()) + active_coordinates = torch.nonzero(active_mask, as_tuple=False) + worst_token_index = [int(item) for item in active_coordinates[worst_flat_index].tolist()] + diagnostics = comparison.get("diagnostics") + if not isinstance(diagnostics, Mapping): + return False + if ( + diagnostics.get("max_abs_diff") != max_abs_diff + or diagnostics.get("worst_token_index") != worst_token_index + or marker.get("case_id") != case.case_id + or marker.get("attempt_id") != attempt_dir.name + or marker.get("status") != status + or marker.get("passed") is not passed + or marker.get("active_token_count") != active_count + or marker.get("mismatch_count") != mismatch_count + or marker.get("max_abs_diff") != max_abs_diff + or marker.get("worst_token_index") != worst_token_index + ): + return False + return True + + +def _serialized_tensor_matches(value: Any, tensor: torch.Tensor) -> bool: + if not isinstance(value, Mapping): + return False + return ( + value.get("dtype") == str(tensor.dtype).removeprefix("torch.") + and value.get("shape") == list(tensor.shape) + and value.get("values") == tensor.tolist() + ) + + +def read_json_object(path: Path) -> dict[str, Any]: + value = strict_json_loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"expected a JSON object: {path}") + return value diff --git a/rl_engine/alignment/cross_config/runner.py b/rl_engine/alignment/cross_config/runner.py new file mode 100644 index 00000000..49ce3904 --- /dev/null +++ b/rl_engine/alignment/cross_config/runner.py @@ -0,0 +1,713 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Control plane for paired read-only scoring runs.""" + +from __future__ import annotations + +import importlib +import math +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Mapping, Optional + +import torch + +from rl_engine.alignment.cross_config._execution import ( + ChildScoringError, + ChildSupervisor, + OperatorExecutionError, + PairedRunnerError, + PairedScorer, + RankCompletenessError, + RankScore, + ScorerIdentityError, + ScoringTimeoutError, + canonical_fingerprint, + device_type, + json_safe, + normalized_dtype, + paired_model_state_fingerprints, + scorer_implementation_fingerprint, + scorer_spec, + validate_rank_outputs, + validate_scorer_identity, +) +from rl_engine.alignment.cross_config._provenance import ( + PAIRED_RUNNER_IMPLEMENTATION_FINGERPRINT, + concrete_scorer_spec, + effective_runtime_status, + execution_environment_provenance, + execution_fingerprint, + factory_options_fingerprint, + mapping_target, + runtime_adapter_fingerprint, + score_metadata, + side_provenance, + target_factory_options, +) +from rl_engine.alignment.cross_config._resume import completed_attempt_matches, read_json_object +from rl_engine.alignment.cross_config.artifacts import ArtifactStore +from rl_engine.alignment.cross_config.comparison import compare_score_artifacts +from rl_engine.alignment.cross_config.operators import ResolvedOperatorOverride +from rl_engine.alignment.cross_config.runtime import ( + RuntimeMaterialization, + RuntimeMaterializationError, +) +from rl_engine.alignment.cross_config.schema import ( + AlignmentResult, + CanonicalScoringBatch, + ExperimentCase, + MaterializationStatus, + ScoreArtifact, + ScorerSpec, + ScoreSide, +) +from rl_engine.kernels.semantic_registry import ( + OperatorInstanceProvenance, + OperatorResolution, + operator_implementation_fingerprint, + operator_instance_fingerprint, +) + + +@dataclass(frozen=True) +class PairedRunResult: + """Completed attempt or a validated resume hit.""" + + case_id: str + attempt_id: str + attempt_dir: Path + resumed: bool + rollout_score: Optional[ScoreArtifact] = None + training_score: Optional[ScoreArtifact] = None + alignment: Optional[AlignmentResult] = None + summary: Mapping[str, Any] = field(default_factory=dict) + + +class PairedRunner: + """Supervise paired scorers and publish one append-only attempt.""" + + def __init__( + self, + artifact_store: ArtifactStore, + *, + timeout_seconds: float = 30.0, + start_method: Optional[str] = None, + ): + if not math.isfinite(timeout_seconds) or timeout_seconds <= 0.0: + raise ValueError("timeout_seconds must be finite and greater than zero") + self.artifact_store = artifact_store + self.timeout_seconds = float(timeout_seconds) + self._child_supervisor = ChildSupervisor(start_method) + self.start_method = self._child_supervisor.start_method + + @property + def active_child_pids(self) -> tuple[int, ...]: + return self._child_supervisor.active_child_pids + + def run( + self, + case: ExperimentCase, + materialization: RuntimeMaterialization, + batch: CanonicalScoringBatch, + rollout_scorer: PairedScorer, + training_scorer: PairedScorer, + resolved_override: ResolvedOperatorOverride, + operator_instances: Mapping[str | ScoreSide, Any], + operator_instance_provenance: Mapping[ + str | ScoreSide, + OperatorInstanceProvenance, + ], + *, + operator_factory_options: Optional[Mapping[str | ScoreSide, Mapping[str, Any]]] = None, + strict: bool = True, + timeout_seconds: Optional[float] = None, + resume: bool = True, + ) -> PairedRunResult: + """Run both sides against one canonical batch and persist the comparison.""" + + deadline_seconds = self.timeout_seconds if timeout_seconds is None else timeout_seconds + if not math.isfinite(deadline_seconds) or deadline_seconds <= 0.0: + raise ValueError("timeout_seconds must be finite and greater than zero") + self._validate_case_inputs(case, materialization, batch) + rollout_spec = scorer_spec(rollout_scorer, ScoreSide.ROLLOUT) + training_spec = scorer_spec(training_scorer, ScoreSide.TRAINING) + specs = {"rollout": rollout_spec, "training": training_spec} + validate_scorer_identity(specs, batch) + model_state_fingerprints = paired_model_state_fingerprints( + rollout_scorer, + training_scorer, + ) + scorer_implementation_fingerprints = { + "rollout": scorer_implementation_fingerprint(rollout_scorer), + "training": scorer_implementation_fingerprint(training_scorer), + } + resolutions, instances, instance_provenance = _validate_exact_operators( + materialization, + resolved_override, + operator_instances, + operator_instance_provenance, + operator_factory_options=operator_factory_options, + specs=specs, + strict=strict, + ) + environment = execution_environment_provenance( + specs, + runtime_adapter_fingerprint=runtime_adapter_fingerprint(materialization), + operator_implementation_fingerprints={ + target: instance_provenance[target].implementation_fingerprint + for target in ("rollout", "training") + }, + ) + environment_fingerprint = canonical_fingerprint(environment) + _require_materialization_executable(materialization, strict=strict) + current_execution_fingerprint = execution_fingerprint( + materialization, + specs=specs, + instance_provenance=instance_provenance, + operator_factory_options=operator_factory_options, + model_state_fingerprints=model_state_fingerprints, + scorer_implementation_fingerprints=scorer_implementation_fingerprints, + environment=environment, + ) + + if resume: + completed = self.artifact_store.completed_attempt(case.experiment_id, case.case_id) + if completed is not None and completed_attempt_matches( + completed, + case, + batch, + materialization=materialization, + specs=specs, + instance_provenance=instance_provenance, + operator_factory_options=operator_factory_options, + model_state_fingerprints=model_state_fingerprints, + scorer_implementation_fingerprints=scorer_implementation_fingerprints, + environment=environment, + execution_fingerprint=current_execution_fingerprint, + ): + summary = read_json_object(completed / "COMPLETE") + return PairedRunResult( + case_id=case.case_id, + attempt_id=completed.name, + attempt_dir=completed, + resumed=True, + summary=summary, + ) + + attempt_dir = self.artifact_store.create_attempt(case.experiment_id, case.case_id) + attempt_id = attempt_dir.name + self._write_attempt_inputs(attempt_dir, attempt_id, case, materialization, batch) + + child_results = self._child_supervisor.run( + attempt_dir, + batch, + batch_size=materialization.binding.batch_size, + scorers={ + "rollout": rollout_scorer, + "training": training_scorer, + }, + specs=specs, + instances=instances, + timeout_seconds=float(deadline_seconds), + ) + rollout_ranks = validate_rank_outputs( + child_results["rollout"], + rollout_spec, + expected_shape=batch.input_ids.shape, + target="rollout", + ) + training_ranks = validate_rank_outputs( + child_results["training"], + training_spec, + expected_shape=batch.input_ids.shape, + target="training", + ) + + rollout_provenance = side_provenance( + materialization.provenance, + resolutions["rollout"], + instance_provenance["rollout"], + child_results["rollout"], + rollout_spec, + status=effective_runtime_status(materialization), + factory_options=target_factory_options(operator_factory_options, "rollout"), + model_state_fingerprint=model_state_fingerprints["rollout"], + scorer_implementation_fingerprint=scorer_implementation_fingerprints["rollout"], + ) + training_provenance = side_provenance( + materialization.provenance, + resolutions["training"], + instance_provenance["training"], + child_results["training"], + training_spec, + status=effective_runtime_status(materialization), + factory_options=target_factory_options(operator_factory_options, "training"), + model_state_fingerprint=model_state_fingerprints["training"], + scorer_implementation_fingerprint=scorer_implementation_fingerprints["training"], + ) + rollout_artifact = ScoreArtifact( + case_id=case.case_id, + attempt_id=attempt_id, + side=ScoreSide.ROLLOUT, + identity=batch.identity, + scorer=concrete_scorer_spec( + rollout_spec, + instance_provenance["rollout"], + ), + selected_logprobs=rollout_ranks[0].selected_logprobs, + active_mask=batch.active_mask, + provenance=rollout_provenance, + ) + training_artifact = ScoreArtifact( + case_id=case.case_id, + attempt_id=attempt_id, + side=ScoreSide.TRAINING, + identity=batch.identity, + scorer=concrete_scorer_spec( + training_spec, + instance_provenance["training"], + ), + selected_logprobs=training_ranks[0].selected_logprobs, + active_mask=batch.active_mask, + provenance=training_provenance, + ) + alignment = compare_score_artifacts(rollout_artifact, training_artifact) + self._write_attempt_results( + attempt_dir, + rollout_artifact, + training_artifact, + alignment, + execution_fingerprint=current_execution_fingerprint, + environment=environment, + environment_fingerprint=environment_fingerprint, + ) + summary = { + "schema_version": "cross_config.complete.v1", + "case_id": case.case_id, + "attempt_id": attempt_id, + "status": alignment.status.value, + "comparable": alignment.comparable, + "passed": alignment.passed, + "active_token_count": alignment.active_token_count, + "mismatch_count": alignment.mismatch_count, + "worst_token_index": alignment.diagnostics.get("worst_token_index"), + "max_abs_diff": alignment.diagnostics.get("max_abs_diff"), + "rollout_backend": instance_provenance["rollout"].backend_id, + "training_backend": instance_provenance["training"].backend_id, + "execution_fingerprint": current_execution_fingerprint, + "environment_fingerprint": environment_fingerprint, + "runner_implementation_fingerprint": PAIRED_RUNNER_IMPLEMENTATION_FINGERPRINT, + } + self.artifact_store.complete_attempt(attempt_dir, summary=summary) + return PairedRunResult( + case_id=case.case_id, + attempt_id=attempt_id, + attempt_dir=attempt_dir, + resumed=False, + rollout_score=rollout_artifact, + training_score=training_artifact, + alignment=alignment, + summary=summary, + ) + + @staticmethod + def _validate_case_inputs( + case: ExperimentCase, + materialization: RuntimeMaterialization, + batch: CanonicalScoringBatch, + ) -> None: + materialized_case = materialization.materialized_case.case + if materialized_case != case: + raise ValueError("materialization case does not exactly match the requested case") + if batch.identity != case.identity: + raise ValueError("canonical scoring batch identity does not match the case identity") + if batch.input_ids.shape[0] < 1: + raise ValueError("canonical scoring batch must contain at least one sequence") + + def _write_attempt_inputs( + self, + attempt_dir: Path, + attempt_id: str, + case: ExperimentCase, + materialization: RuntimeMaterialization, + batch: CanonicalScoringBatch, + ) -> None: + envelope = {"case_id": case.case_id, "attempt_id": attempt_id} + self.artifact_store.write_json( + attempt_dir, + "requested", + {**envelope, "schema_version": "cross_config.requested.v1", "case": case.to_dict()}, + ) + self.artifact_store.write_json( + attempt_dir, + "materialized", + { + **envelope, + "schema_version": "cross_config.materialized_envelope.v1", + "materialized_case": materialization.materialized_case.to_dict(), + }, + ) + self.artifact_store.write_json( + attempt_dir, + "identity", + { + **envelope, + "schema_version": "cross_config.identity_envelope.v1", + "identity": batch.identity.to_dict(), + }, + ) + + def _write_attempt_results( + self, + attempt_dir: Path, + rollout: ScoreArtifact, + training: ScoreArtifact, + alignment: AlignmentResult, + *, + execution_fingerprint: str, + environment: Mapping[str, Any], + environment_fingerprint: str, + ) -> None: + self.artifact_store.write_json( + attempt_dir, + "actual", + { + "case_id": rollout.case_id, + "attempt_id": rollout.attempt_id, + "schema_version": "cross_config.actual.v1", + "execution_fingerprint": execution_fingerprint, + "environment": environment, + "environment_fingerprint": environment_fingerprint, + "runner_implementation_fingerprint": PAIRED_RUNNER_IMPLEMENTATION_FINGERPRINT, + "operator_source": "exact_resolution_and_instance", + "rollout": rollout.provenance.to_dict(), + "training": training.provenance.to_dict(), + }, + ) + self.artifact_store.write_tensor_bundle( + attempt_dir, + "score_rollout", + { + "selected_logprobs": rollout.selected_logprobs, + "active_mask": rollout.active_mask, + }, + metadata=score_metadata(rollout), + ) + self.artifact_store.write_tensor_bundle( + attempt_dir, + "score_training", + { + "selected_logprobs": training.selected_logprobs, + "active_mask": training.active_mask, + }, + metadata=score_metadata(training), + ) + self.artifact_store.write_json( + attempt_dir, + "comparison", + alignment.to_dict(), + ) + token_artifact = alignment.token_artifact + if token_artifact is None: + empty = torch.empty((0,), dtype=torch.float32) + token_tensors = { + "rollout_logprobs": empty, + "training_logprobs": empty, + "active_mask": torch.empty((0,), dtype=torch.bool), + "absolute_diff": empty, + "mismatch_mask": torch.empty((0,), dtype=torch.bool), + } + else: + token_tensors = { + "rollout_logprobs": token_artifact.rollout_logprobs, + "training_logprobs": token_artifact.training_logprobs, + "active_mask": token_artifact.active_mask, + "absolute_diff": token_artifact.absolute_diff, + "mismatch_mask": token_artifact.mismatch_mask, + } + self.artifact_store.write_tensor_bundle( + attempt_dir, + "token_diffs", + token_tensors, + metadata={ + "case_id": alignment.case_id, + "attempt_id": alignment.attempt_id, + "status": alignment.status.value, + "fixed_threshold": alignment.fixed_threshold, + }, + ) + + +def _validate_exact_operators( + materialization: RuntimeMaterialization, + resolved: ResolvedOperatorOverride, + instances: Mapping[str | ScoreSide, Any], + instance_provenance: Mapping[str | ScoreSide, OperatorInstanceProvenance], + *, + operator_factory_options: Optional[Mapping[str | ScoreSide, Mapping[str, Any]]], + specs: Mapping[str, ScorerSpec], + strict: bool, +) -> tuple[ + dict[str, OperatorResolution], + dict[str, Any], + dict[str, OperatorInstanceProvenance], +]: + if resolved.semantic_op != "selected_logprob": + raise OperatorExecutionError("PairedRunner V1 requires semantic_op='selected_logprob'") + resolutions: dict[str, OperatorResolution] = {} + concrete_instances: dict[str, Any] = {} + provenance: dict[str, OperatorInstanceProvenance] = {} + for target in ("rollout", "training"): + resolution = resolved.for_target(target) # type: ignore[arg-type] + if resolution is None: + raise OperatorExecutionError(f"missing exact {target} operator resolution") + if resolution.target != target: + raise OperatorExecutionError( + f"{target} operator resolution reports target={resolution.target!r}" + ) + if ( + resolution.descriptor.semantic_op != "selected_logprob" + or resolution.trace.semantic_op != "selected_logprob" + ): + raise OperatorExecutionError(f"{target} resolution does not describe selected_logprob") + if resolution.trace.status != "resolved" or resolution.trace.concrete_backend is None: + raise OperatorExecutionError( + f"{target} operator is not exactly observable: {resolution.trace.status}" + ) + if resolution.trace.fallback_attempts: + raise OperatorExecutionError(f"{target} operator resolution attempted fallback") + if strict and not resolution.strict: + raise OperatorExecutionError(f"{target} operator was not resolved in strict mode") + if resolution.trace.concrete_backend != resolution.descriptor.backend_id: + raise OperatorExecutionError(f"{target} resolution backend evidence is inconsistent") + if resolution.trace.descriptor_fingerprint != resolution.descriptor.descriptor_fingerprint: + raise OperatorExecutionError( + f"{target} resolution descriptor fingerprint is inconsistent" + ) + if device_type(resolution.requirements.device) != device_type(specs[target].device): + raise OperatorExecutionError( + f"{target} operator resolution device does not match scorer device" + ) + if normalized_dtype(resolution.requirements.dtype) != normalized_dtype(specs[target].dtype): + raise OperatorExecutionError(f"{target} resolution dtype does not match scorer dtype") + _validate_exact_topology( + materialization, + resolution, + specs[target], + target=target, + ) + instance = mapping_target(instances, target) + if instance is None: + raise OperatorExecutionError(f"missing instantiated {target} operator") + _require_instance_matches_resolution(resolution, instance, target=target) + instance_evidence = mapping_target(instance_provenance, target) + if not isinstance(instance_evidence, OperatorInstanceProvenance): + raise OperatorExecutionError(f"missing sealed {target} operator instance provenance") + _validate_instance_provenance( + resolution, + instance, + instance_evidence, + factory_options=target_factory_options(operator_factory_options, target), + target=target, + ) + if instance_evidence.backend_id != resolution.trace.concrete_backend: + raise OperatorExecutionError(f"{target} instance backend does not match resolution") + declared = materialization.binding.operator_backends.get(target) + if declared is not None and declared != instance_evidence.backend_id: + raise OperatorExecutionError( + f"{target} exact backend {instance_evidence.backend_id!r} does not match " + f"declared override {declared!r}" + ) + resolutions[target] = resolution + concrete_instances[target] = instance + provenance[target] = instance_evidence + requested_logp = materialization.materialized_case.case.requested.get("logp") + requested_backend = ( + requested_logp.get("backend") if isinstance(requested_logp, Mapping) else None + ) + if requested_backend != provenance["rollout"].backend_id: + raise OperatorExecutionError( + "exact rollout operator does not match the public logp.backend request: " + f"{provenance['rollout'].backend_id!r} != {requested_backend!r}" + ) + return resolutions, concrete_instances, provenance + + +def _require_materialization_executable( + materialization: RuntimeMaterialization, + *, + strict: bool, +) -> None: + rejected = { + MaterializationStatus.ERROR, + MaterializationStatus.UNSUPPORTED, + MaterializationStatus.UNOBSERVABLE, + } + if strict: + rejected.add(MaterializationStatus.FALLBACK) + if not materialization.applications and materialization.materialized_case.status in rejected: + raise RuntimeMaterializationError( + f"case {materialization.materialized_case.case.case_id} is not executable: " + f"materialization={materialization.materialized_case.status.value}" + ) + problems = [] + for application in materialization.applications: + if ( + application.path == "logp.backend" + and application.status is MaterializationStatus.UNOBSERVABLE + ): + continue + if application.status in rejected: + problems.append( + f"{application.path}={application.status.value}: " + f"{application.evidence.get('reason', 'no evidence')}" + ) + if problems: + raise RuntimeMaterializationError( + f"case {materialization.materialized_case.case.case_id} is not executable: " + + "; ".join(problems) + ) + + +def _validate_exact_topology( + materialization: RuntimeMaterialization, + resolution: OperatorResolution, + spec: ScorerSpec, + *, + target: str, +) -> None: + bound_topology = mapping_target(materialization.binding.topology, target) + if not isinstance(bound_topology, Mapping): + raise OperatorExecutionError(f"{target} materialized topology is missing") + expected = dict(bound_topology) + topology_paths = { + "rollout": ( + ("rollout.tensor_parallel_size", "tensor_parallel_size"), + ("rollout.context_parallel_size", "context_parallel_size"), + ), + "training": (("training.sharding", "sharding"),), + } + required_keys = {"world_size", *(key for _, key in topology_paths[target])} + missing_keys = sorted(required_keys.difference(expected)) + if missing_keys: + raise OperatorExecutionError( + f"{target} materialized topology is missing required keys: {missing_keys!r}" + ) + if expected.get("world_size") != spec.world_size: + raise OperatorExecutionError( + f"{target} scorer world_size does not match materialized topology" + ) + if dict(resolution.requirements.topology) != expected: + raise OperatorExecutionError( + f"{target} resolution topology does not match materialized topology" + ) + if dict(spec.topology) != expected: + raise OperatorExecutionError( + f"{target} scorer topology does not match materialized topology" + ) + + actual_by_path = { + application.path: application.actual for application in materialization.applications + } + for path, key in topology_paths[target]: + if actual_by_path.get(path) != expected[key]: + raise OperatorExecutionError( + f"{target} actual {path} does not match exact operator topology" + ) + + +def _require_instance_matches_resolution( + resolution: OperatorResolution, + instance: Any, + *, + target: str, +) -> None: + implementation = resolution.descriptor.implementation_class_or_factory + factory = implementation + if isinstance(implementation, str): + try: + module_name, object_name = implementation.rsplit(".", 1) + factory = getattr(importlib.import_module(module_name), object_name) + except (ValueError, ImportError, AttributeError, ModuleNotFoundError) as exc: + raise OperatorExecutionError( + f"{target} exact operator factory cannot be verified: {exc}" + ) from exc + if isinstance(factory, type) and not isinstance(instance, factory): + raise OperatorExecutionError( + f"{target} operator instance type {type(instance).__qualname__!r} " + f"does not match resolved factory {factory.__qualname__!r}" + ) + if not callable(instance) and not callable(getattr(instance, "apply_fp32", None)): + raise OperatorExecutionError( + f"{target} selected-logprob operator instance is not executable" + ) + + +def _validate_instance_provenance( + resolution: OperatorResolution, + instance: Any, + provenance: OperatorInstanceProvenance, + *, + factory_options: Mapping[str, Any], + target: str, +) -> None: + expected_concrete = f"{type(instance).__module__}.{type(instance).__qualname__}" + expected_factory = resolution.descriptor.implementation_reference + if expected_factory is None: + raise OperatorExecutionError(f"{target} resolved operator has no factory reference") + implementation = resolution.descriptor.implementation_class_or_factory + if implementation is None: + raise OperatorExecutionError(f"{target} resolved operator has no implementation") + mismatches = [] + if provenance.semantic_op != resolution.descriptor.semantic_op: + mismatches.append("semantic_op") + if provenance.backend_id != resolution.descriptor.backend_id: + mismatches.append("backend_id") + if provenance.target != target: + mismatches.append("target") + if provenance.factory_reference != expected_factory: + mismatches.append("factory_reference") + if provenance.concrete_implementation != expected_concrete: + mismatches.append("concrete_implementation") + if provenance.descriptor_fingerprint != resolution.descriptor.descriptor_fingerprint: + mismatches.append("descriptor_fingerprint") + observed_implementation_fingerprint = operator_implementation_fingerprint( + implementation, + instance, + ) + if provenance.implementation_fingerprint != observed_implementation_fingerprint: + mismatches.append("implementation_fingerprint") + + if json_safe(provenance.factory_options) != json_safe(factory_options): + mismatches.append("factory_options") + if provenance.factory_options_fingerprint != factory_options_fingerprint(factory_options): + mismatches.append("factory_options_fingerprint") + expected_instance_fingerprint = operator_instance_fingerprint( + descriptor_fingerprint=resolution.descriptor.descriptor_fingerprint, + factory_reference=expected_factory, + concrete_implementation=expected_concrete, + implementation_fingerprint=observed_implementation_fingerprint, + factory_options_fingerprint=factory_options_fingerprint(factory_options), + ) + if provenance.instance_fingerprint != expected_instance_fingerprint: + mismatches.append("instance_fingerprint") + if mismatches: + raise OperatorExecutionError( + f"{target} operator instance provenance is inconsistent: " + ", ".join(mismatches) + ) + + +__all__ = [ + "ChildScoringError", + "OperatorExecutionError", + "PairedRunResult", + "PairedRunner", + "PairedRunnerError", + "PairedScorer", + "RankCompletenessError", + "RankScore", + "ScorerIdentityError", + "ScoringTimeoutError", +] From b3171a8570c3536d254871256d385a4e93187823 Mon Sep 17 00:00:00 2001 From: CyberSecurityErial <2710555967@qq.com> Date: Sun, 19 Jul 2026 08:34:41 +0800 Subject: [PATCH 6/8] feat(alignment): add CPU smoke workflow and CLI --- .gitignore | 3 + examples/cross_config_s0_cpu_smoke.json | 75 ++ .../cross_config_s1_distributed_smoke.json | 98 +++ examples/cross_config_s2_vllm_tp_vs_fsdp.json | 128 ++++ ...cross_config_s3_qwen3_8b_tp4_cp4_bf16.json | 134 ++++ rl_engine/alignment/cross_config/__main__.py | 138 ++++ rl_engine/alignment/testing/__init__.py | 18 + .../alignment/testing/cpu_cross_config.py | 696 ++++++++++++++++++ .../testing/smoke_ops/SMOKE_OPERATORS.md | 30 + .../alignment/testing/smoke_ops/__init__.py | 75 ++ .../smoke_ops/smoke_only_logp_offset.py | 93 +++ .../smoke_ops/smoke_only_logp_reference.py | 108 +++ 12 files changed, 1596 insertions(+) create mode 100644 examples/cross_config_s0_cpu_smoke.json create mode 100644 examples/cross_config_s1_distributed_smoke.json create mode 100644 examples/cross_config_s2_vllm_tp_vs_fsdp.json create mode 100644 examples/cross_config_s3_qwen3_8b_tp4_cp4_bf16.json create mode 100644 rl_engine/alignment/cross_config/__main__.py create mode 100644 rl_engine/alignment/testing/__init__.py create mode 100644 rl_engine/alignment/testing/cpu_cross_config.py create mode 100644 rl_engine/alignment/testing/smoke_ops/SMOKE_OPERATORS.md create mode 100644 rl_engine/alignment/testing/smoke_ops/__init__.py create mode 100644 rl_engine/alignment/testing/smoke_ops/smoke_only_logp_offset.py create mode 100644 rl_engine/alignment/testing/smoke_ops/smoke_only_logp_reference.py diff --git a/.gitignore b/.gitignore index ae89c0d7..3f4ea52e 100644 --- a/.gitignore +++ b/.gitignore @@ -206,3 +206,6 @@ cython_debug/ marimo/_static/ marimo/_lsp/ __marimo__/ + +# Cross-configuration alignment local run artifacts +/runs/ diff --git a/examples/cross_config_s0_cpu_smoke.json b/examples/cross_config_s0_cpu_smoke.json new file mode 100644 index 00000000..0720c67d --- /dev/null +++ b/examples/cross_config_s0_cpu_smoke.json @@ -0,0 +1,75 @@ +{ + "schema_version": "cross_config.experiment_config.v1", + "experiment_id": "cross-config-s0-cpu-smoke-v1", + "scenario_id": "cross_config.s0.cpu_smoke.v1", + "contract_source": "ws1", + "contract_version": "current", + "strategy": "one_at_a_time", + "strict_fallback": true, + "identity": { + "checkpoint_id": "cross_config.synthetic.cpu_logits.v1", + "model_version": "immutable:cross-config-synthetic-cpu-v1", + "tokenizer_id": "cross_config.synthetic_tokenizer.v1", + "tokenizer_policy": "pretokenized_fixture.v1;add_special_tokens=false;padding_side=right", + "token_ids": [ + [11, 12, 13, 21, 22, 23], + [31, 32, 33, 41, 42, 43] + ], + "selected_token_ids": [ + [11, 12, 13, 21, 22, 23], + [31, 32, 33, 41, 42, 43] + ], + "active_mask": [ + [false, false, false, true, true, true], + [false, false, false, true, true, true] + ], + "attention_mask": [ + [true, true, true, true, true, true], + [true, true, true, true, true, true] + ], + "position_ids": [ + [0, 1, 2, 3, 4, 5], + [0, 1, 2, 3, 4, 5] + ], + "pre_update_state": "synthetic_read_only:no_parameters:no_optimizer", + "cache_metadata": { + "use_cache": false + }, + "packing_metadata": { + "packed": false + } + }, + "baseline": { + "batch": { + "size": 2 + }, + "rollout": { + "tensor_parallel_size": 1, + "context_parallel_size": 1, + "dtype": "float32", + "enable_prefix_caching": false, + "enforce_eager": true + }, + "training": { + "attention_backend": "eager", + "compute_dtype": "float32", + "sharding": "unsharded" + }, + "logp": { + "backend": "smoke_only.logp_reference" + } + }, + "interventions": [], + "operators": { + "selected_logprob": { + "rollout": "smoke_only.logp_reference", + "training": "smoke_only.logp_reference" + } + }, + "scenario": { + "level": "S0", + "name": "CPU framework smoke", + "device": "cpu", + "hardware_required": false + } +} diff --git a/examples/cross_config_s1_distributed_smoke.json b/examples/cross_config_s1_distributed_smoke.json new file mode 100644 index 00000000..e7112c0d --- /dev/null +++ b/examples/cross_config_s1_distributed_smoke.json @@ -0,0 +1,98 @@ +{ + "schema_version": "cross_config.experiment_config.v1", + "experiment_id": "cross-config-s1-distributed-smoke-v1", + "scenario_id": "cross_config.s1.distributed_smoke.v1", + "contract_source": "ws1", + "contract_version": "current", + "strategy": "one_at_a_time", + "strict_fallback": true, + "identity": { + "checkpoint_id": "Qwen/Qwen3-0.6B", + "model_version": "c1899de289a04d12100db370d81485cdf75e47ca", + "tokenizer_id": "Qwen/Qwen3-0.6B@c1899de289a04d12100db370d81485cdf75e47ca", + "tokenizer_policy": "pretokenized_fixture.v1;add_special_tokens=false;padding_side=left", + "token_ids": [ + [151643, 8948, 198, 11, 12, 13, 14, 15], + [151643, 8948, 198, 21, 22, 23, 24, 25] + ], + "selected_token_ids": [ + [151643, 8948, 198, 11, 12, 13, 14, 15], + [151643, 8948, 198, 21, 22, 23, 24, 25] + ], + "active_mask": [ + [false, false, false, false, true, true, true, true], + [false, false, false, false, true, true, true, true] + ], + "attention_mask": [ + [true, true, true, true, true, true, true, true], + [true, true, true, true, true, true, true, true] + ], + "position_ids": [ + [0, 1, 2, 3, 4, 5, 6, 7], + [0, 1, 2, 3, 4, 5, 6, 7] + ], + "pre_update_state": "checkpoint_revision:c1899de289a04d12100db370d81485cdf75e47ca;optimizer_steps=0", + "cache_metadata": { + "position_policy": "absolute" + }, + "packing_metadata": { + "packed": false + } + }, + "baseline": { + "batch": { + "size": 2 + }, + "rollout": { + "tensor_parallel_size": 1, + "context_parallel_size": 1, + "dtype": "bfloat16", + "enable_prefix_caching": true, + "enforce_eager": false + }, + "training": { + "attention_backend": "sdpa", + "compute_dtype": "bfloat16", + "sharding": "unsharded" + }, + "logp": { + "backend": "native" + } + }, + "interventions": [ + { + "path": "batch.size", + "values": [ + 1 + ] + }, + { + "path": "rollout.tensor_parallel_size", + "values": [ + 2 + ] + }, + { + "path": "rollout.context_parallel_size", + "values": [ + 2 + ] + }, + { + "path": "training.sharding", + "values": [ + "fsdp" + ] + } + ], + "scenario": { + "level": "S1", + "name": "Smallest distributed lifecycle smoke", + "device": "cuda", + "hardware_required": true, + "model_id": "Qwen/Qwen3-0.6B", + "model_revision": "c1899de289a04d12100db370d81485cdf75e47ca", + "rollout_engine": "vllm", + "training_engine": "fsdp_score_only" + } +} diff --git a/examples/cross_config_s2_vllm_tp_vs_fsdp.json b/examples/cross_config_s2_vllm_tp_vs_fsdp.json new file mode 100644 index 00000000..10a44cf3 --- /dev/null +++ b/examples/cross_config_s2_vllm_tp_vs_fsdp.json @@ -0,0 +1,128 @@ +{ + "schema_version": "cross_config.experiment_config.v1", + "experiment_id": "cross-config-s2-vllm-tp-vs-fsdp-v1", + "scenario_id": "cross_config.s2.vllm_tp_vs_fsdp.v1", + "contract_source": "ws1", + "contract_version": "current", + "strategy": "one_at_a_time", + "strict_fallback": true, + "identity": { + "checkpoint_id": "Qwen/Qwen3-8B", + "model_version": "b968826d9c46dd6066d109eabc6255188de91218", + "tokenizer_id": "Qwen/Qwen3-8B@b968826d9c46dd6066d109eabc6255188de91218", + "tokenizer_policy": "pretokenized_fixture.v1;add_special_tokens=false;padding_side=left", + "token_ids": [ + [151643, 8948, 198, 11, 12, 13, 14, 15], + [151643, 8948, 198, 21, 22, 23, 24, 25] + ], + "selected_token_ids": [ + [151643, 8948, 198, 11, 12, 13, 14, 15], + [151643, 8948, 198, 21, 22, 23, 24, 25] + ], + "active_mask": [ + [false, false, false, false, true, true, true, true], + [false, false, false, false, true, true, true, true] + ], + "attention_mask": [ + [true, true, true, true, true, true, true, true], + [true, true, true, true, true, true, true, true] + ], + "position_ids": [ + [0, 1, 2, 3, 4, 5, 6, 7], + [0, 1, 2, 3, 4, 5, 6, 7] + ], + "pre_update_state": "checkpoint_revision:b968826d9c46dd6066d109eabc6255188de91218;optimizer_steps=0", + "cache_metadata": { + "position_policy": "absolute" + }, + "packing_metadata": { + "packed": false + } + }, + "baseline": { + "batch": { + "size": 2 + }, + "rollout": { + "tensor_parallel_size": 2, + "context_parallel_size": 1, + "dtype": "bfloat16", + "enable_prefix_caching": true, + "enforce_eager": false + }, + "training": { + "attention_backend": "flash_attention_2", + "compute_dtype": "bfloat16", + "sharding": "fsdp" + }, + "logp": { + "backend": "native" + } + }, + "interventions": [ + { + "path": "batch.size", + "values": [ + 1 + ] + }, + { + "path": "rollout.tensor_parallel_size", + "values": [ + 1 + ] + }, + { + "path": "rollout.dtype", + "values": [ + "float32" + ] + }, + { + "path": "rollout.enable_prefix_caching", + "values": [ + false + ] + }, + { + "path": "rollout.enforce_eager", + "values": [ + true + ] + }, + { + "path": "training.attention_backend", + "values": [ + "eager" + ] + }, + { + "path": "training.compute_dtype", + "values": [ + "float32" + ] + }, + { + "path": "logp.backend", + "values": [ + "rlkernel.reference_logp" + ] + }, + { + "path": "training.sharding", + "values": [ + "unsharded" + ] + } + ], + "scenario": { + "level": "S2", + "name": "Issue 111 vLLM TP=2 versus training FSDP", + "device": "cuda", + "hardware_required": true, + "model_id": "Qwen/Qwen3-8B", + "model_revision": "b968826d9c46dd6066d109eabc6255188de91218", + "rollout_engine": "vllm", + "training_engine": "fsdp_score_only" + } +} diff --git a/examples/cross_config_s3_qwen3_8b_tp4_cp4_bf16.json b/examples/cross_config_s3_qwen3_8b_tp4_cp4_bf16.json new file mode 100644 index 00000000..2dcf79fb --- /dev/null +++ b/examples/cross_config_s3_qwen3_8b_tp4_cp4_bf16.json @@ -0,0 +1,134 @@ +{ + "schema_version": "cross_config.experiment_config.v1", + "experiment_id": "cross-config-s3-qwen3-8b-tp4-cp4-bf16-v1", + "scenario_id": "cross_config.s3.qwen3_8b_tp4_cp4_bf16.v1", + "contract_source": "ws1", + "contract_version": "current", + "strategy": "one_at_a_time", + "strict_fallback": true, + "identity": { + "checkpoint_id": "Qwen/Qwen3-8B", + "model_version": "b968826d9c46dd6066d109eabc6255188de91218", + "tokenizer_id": "Qwen/Qwen3-8B@b968826d9c46dd6066d109eabc6255188de91218", + "tokenizer_policy": "pretokenized_fixture.v1;add_special_tokens=false;padding_side=left", + "token_ids": [ + [151643, 8948, 198, 11, 12, 13, 14, 15], + [151643, 8948, 198, 21, 22, 23, 24, 25] + ], + "selected_token_ids": [ + [151643, 8948, 198, 11, 12, 13, 14, 15], + [151643, 8948, 198, 21, 22, 23, 24, 25] + ], + "active_mask": [ + [false, false, false, false, true, true, true, true], + [false, false, false, false, true, true, true, true] + ], + "attention_mask": [ + [true, true, true, true, true, true, true, true], + [true, true, true, true, true, true, true, true] + ], + "position_ids": [ + [0, 1, 2, 3, 4, 5, 6, 7], + [0, 1, 2, 3, 4, 5, 6, 7] + ], + "pre_update_state": "checkpoint_revision:b968826d9c46dd6066d109eabc6255188de91218;optimizer_steps=0", + "cache_metadata": { + "position_policy": "absolute" + }, + "packing_metadata": { + "packed": false + } + }, + "baseline": { + "batch": { + "size": 2 + }, + "rollout": { + "tensor_parallel_size": 4, + "context_parallel_size": 4, + "dtype": "bfloat16", + "enable_prefix_caching": true, + "enforce_eager": false + }, + "training": { + "attention_backend": "flash_attention_2", + "compute_dtype": "bfloat16", + "sharding": "fsdp" + }, + "logp": { + "backend": "native" + } + }, + "interventions": [ + { + "path": "batch.size", + "values": [ + 1 + ] + }, + { + "path": "rollout.tensor_parallel_size", + "values": [ + 1 + ] + }, + { + "path": "rollout.context_parallel_size", + "values": [ + 1 + ] + }, + { + "path": "rollout.dtype", + "values": [ + "float32" + ] + }, + { + "path": "rollout.enable_prefix_caching", + "values": [ + false + ] + }, + { + "path": "rollout.enforce_eager", + "values": [ + true + ] + }, + { + "path": "training.attention_backend", + "values": [ + "eager" + ] + }, + { + "path": "training.compute_dtype", + "values": [ + "float32" + ] + }, + { + "path": "logp.backend", + "values": [ + "rlkernel.reference_logp" + ] + }, + { + "path": "training.sharding", + "values": [ + "unsharded" + ] + } + ], + "scenario": { + "level": "S3", + "name": "Roadmap Qwen3-8B TP=4 CP=4 BF16 milestone", + "device": "cuda", + "hardware_required": true, + "model_id": "Qwen/Qwen3-8B", + "model_revision": "b968826d9c46dd6066d109eabc6255188de91218", + "rollout_engine": "vllm", + "training_engine": "fsdp_score_only" + } +} diff --git a/rl_engine/alignment/cross_config/__main__.py b/rl_engine/alignment/cross_config/__main__.py new file mode 100644 index 00000000..cd047ca3 --- /dev/null +++ b/rl_engine/alignment/cross_config/__main__.py @@ -0,0 +1,138 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Command-line interface for cross-configuration experiments.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any, Optional, Sequence + +from rl_engine.alignment.cross_config.artifacts import ArtifactStore +from rl_engine.alignment.cross_config.config import ExperimentConfig, load_config +from rl_engine.alignment.cross_config.execution_plan import build_execution_plan + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + + plan = commands.add_parser("plan", help="validate and persist a plan without execution") + _add_common_arguments(plan) + + run = commands.add_parser("run", help="execute a plan with an explicit runtime adapter") + _add_common_arguments(run) + run.add_argument( + "--runtime", + required=True, + choices=("cpu-smoke",), + help="Runtime adapter; only the temporary CPU smoke adapter ships in V1", + ) + run.add_argument( + "--allow-smoke-operators", + action="store_true", + help="Explicitly authorize temporary smoke-only operator backends", + ) + run.add_argument( + "--timeout-seconds", + type=float, + default=30.0, + help="Per paired-scoring attempt deadline", + ) + run.add_argument( + "--no-resume", + action="store_true", + help="Create new attempts even when matching COMPLETE artifacts exist", + ) + return parser + + +def _add_common_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("config", type=Path, help="Versioned experiment JSON") + parser.add_argument( + "--output-root", + type=Path, + default=Path("runs"), + help="Append-only artifact root (default: runs)", + ) + + +def main(argv: Optional[Sequence[str]] = None) -> int: + args = build_parser().parse_args(argv) + try: + config = load_config(args.config) + if args.command == "plan": + summary = record_plan(config, args.output_root) + else: + summary = _run(config, args) + except Exception as exc: + summary = { + "schema_version": "cross_config.cli_summary.v1", + "status": "error", + "error_type": f"{type(exc).__module__}.{type(exc).__qualname__}", + "error": str(exc), + } + print(json.dumps(summary, sort_keys=True)) + print(f"cross-configuration error: {type(exc).__name__}: {exc}", file=sys.stderr) + return 2 + + print(json.dumps(summary, sort_keys=True)) + if args.command == "plan": + print( + f"planned {summary['planned_case_count']} cases; no runtime was created", + file=sys.stderr, + ) + return 0 + print( + f"CPU smoke: {summary['status']} ({len(summary['cases'])} cases)", + file=sys.stderr, + ) + for case in summary["cases"]: + print( + f" {case['case_id']}: {case['status']}; actual backends " + f"rollout={case['rollout_backend']}, training={case['training_backend']}; " + f"worst sample/token={case['worst_token_index']}; " + f"mismatches={case['mismatch_count']}; resumed={case['resumed']}", + file=sys.stderr, + ) + return 0 if summary["status"] == "pass" else 1 + + +def record_plan(config: ExperimentConfig, output_root: Path) -> dict[str, Any]: + plan = build_execution_plan(config) + store = ArtifactStore(output_root) + experiment_dir = store.initialize_experiment( + config.definition.experiment_id, + experiment=plan.experiment, + plan=plan.rows(), + ) + return { + "schema_version": "cross_config.cli_summary.v1", + "status": "planned", + "experiment_id": config.definition.experiment_id, + "scenario_id": config.definition.scenario_id, + "planned_case_count": len(plan.entries), + "planning_issues": [issue.to_dict() for issue in plan.issues], + "artifact_dir": str(experiment_dir), + } + + +def _run(config: ExperimentConfig, args: argparse.Namespace) -> dict[str, Any]: + if args.runtime != "cpu-smoke": # pragma: no cover - argparse owns choices + raise ValueError(f"unsupported runtime {args.runtime!r}") + from rl_engine.alignment.testing.cpu_cross_config import run_cpu_experiment + + return run_cpu_experiment( + config, + output_root=args.output_root, + allow_smoke_operators=args.allow_smoke_operators, + timeout_seconds=args.timeout_seconds, + resume=not args.no_resume, + ) + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/rl_engine/alignment/testing/__init__.py b/rl_engine/alignment/testing/__init__.py new file mode 100644 index 00000000..2254e26a --- /dev/null +++ b/rl_engine/alignment/testing/__init__.py @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Test-only integration helpers for the alignment framework.""" + +from .smoke_ops import ( + SMOKE_ONLY_LOGP_OFFSET_BACKEND_ID, + SMOKE_ONLY_LOGP_REFERENCE_BACKEND_ID, + register_smoke_operators, + smoke_operator_descriptors, +) + +__all__ = [ + "SMOKE_ONLY_LOGP_OFFSET_BACKEND_ID", + "SMOKE_ONLY_LOGP_REFERENCE_BACKEND_ID", + "register_smoke_operators", + "smoke_operator_descriptors", +] diff --git a/rl_engine/alignment/testing/cpu_cross_config.py b/rl_engine/alignment/testing/cpu_cross_config.py new file mode 100644 index 00000000..9c9e2ce2 --- /dev/null +++ b/rl_engine/alignment/testing/cpu_cross_config.py @@ -0,0 +1,696 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""CPU-only adapters for cross-configuration smoke execution. + +This module is deliberately outside the production framework package. It gives +the CLI and tests a deterministic execution target without implying CUDA, +distributed, vLLM, or training-runtime support. +""" + +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace +from typing import Any, Mapping, Optional + +import torch + +from rl_engine.alignment.cross_config.artifacts import ArtifactStore +from rl_engine.alignment.cross_config.config import ExperimentConfig, OperatorSelection +from rl_engine.alignment.cross_config.execution_plan import build_execution_plan +from rl_engine.alignment.cross_config.operators import ( + OperatorBridge, + OperatorOverride, + selected_logprobs_with_operator, +) +from rl_engine.alignment.cross_config.runner import PairedRunner, PairedRunResult, RankScore +from rl_engine.alignment.cross_config.runtime import ( + AdapterMaterialization, + KnobApplication, + RuntimeBinding, + RuntimeTools, +) +from rl_engine.alignment.cross_config.schema import ( + CanonicalScoringBatch, + ExperimentCase, + KnobDescriptor, + MaterializationStatus, + ScorerSpec, + ScoreSide, +) +from rl_engine.executors.stateless_executor import ( + StatelessForwardConfig, + StatelessForwardExecutor, + StatelessForwardInputs, +) +from rl_engine.kernels.registry import kernel_registry +from rl_engine.kernels.semantic_registry import ( + OperatorRequirements, + OperatorResolutionPolicy, + SemanticOperatorCatalog, +) +from rl_engine.kernels.semantic_registry import ( + implementation_fingerprint as fingerprint_implementation, +) + +CPU_SCORER_IMPLEMENTATION_FINGERPRINT = "cross_config.cpu_stateless_scorer.v1" + + +class SyntheticCpuCausalLM(torch.nn.Module): + """Deterministic parameter-free model for the named CPU smoke scenario.""" + + def __init__(self, vocab_size: int): + super().__init__() + self.vocab_axis: torch.Tensor + self.register_buffer( + "vocab_axis", + torch.arange(vocab_size, dtype=torch.float32), + persistent=False, + ) + self.config = SimpleNamespace(use_cache=False, _attn_implementation="eager") + self.generation_config = SimpleNamespace(use_cache=False) + + def forward( + self, + input_ids: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.Tensor] = None, + use_cache: Optional[bool] = None, + ) -> Any: + del attention_mask + if use_cache not in {None, False}: + raise ValueError("CPU smoke scoring forbids KV-cache generation") + if input_ids.device.type != "cpu": + raise ValueError("the synthetic smoke model accepts CPU tensors only") + if position_ids is None: + position_ids = torch.arange(input_ids.shape[1], device="cpu").expand_as(input_ids) + centers = torch.remainder(input_ids + position_ids + 1, self.vocab_axis.numel()).float() + logits = -torch.abs(self.vocab_axis.view(1, 1, -1) - centers.unsqueeze(-1)) * 0.125 + return SimpleNamespace(logits=logits, past_key_values=None) + + +class CpuStatelessScorer: + """Read-only teacher-forcing adapter over ``StatelessForwardExecutor``.""" + + optimizer = None + implementation_fingerprint = CPU_SCORER_IMPLEMENTATION_FINGERPRINT + + def __init__( + self, + model: torch.nn.Module, + spec: ScorerSpec, + config: Optional[StatelessForwardConfig] = None, + ): + if spec.world_size != 1: + raise ValueError("CpuStatelessScorer supports only world_size=1") + if _device_type(spec.device) != "cpu": + raise ValueError("CpuStatelessScorer is explicitly CPU-only") + resolved_config = config or StatelessForwardConfig( + mode="reference", + attention_backend="eager", + output_dtype=_torch_dtype(spec.dtype), + ) + if resolved_config.mode not in {"reference", "both"}: + raise ValueError("CpuStatelessScorer requires reference scoring mode") + expected_dtype = _torch_dtype(spec.dtype) + if resolved_config.output_dtype is not expected_dtype: + raise ValueError("stateless output_dtype must match the scorer dtype") + _require_module_on_cpu(model) + _require_module_float_dtype(model, expected_dtype) + self.model = model + self.spec = spec + self.config = resolved_config + + def score( + self, + batch: CanonicalScoringBatch, + *, + batch_size: int, + operator: Any, + ) -> tuple[RankScore, ...]: + if batch_size < 1: + raise ValueError("batch_size must be greater than zero") + + def selected_logprob_fn( + logits: torch.Tensor, + token_ids: torch.Tensor, + *, + mask: Optional[torch.Tensor] = None, + temperature: float = 1.0, + output_dtype: torch.dtype = torch.float32, + ) -> torch.Tensor: + return selected_logprobs_with_operator( + operator, + logits, + token_ids, + active_mask=mask, + temperature=temperature, + output_dtype=output_dtype, + ) + + executor = StatelessForwardExecutor( + self.model, + self.config, + selected_logprob_fn=selected_logprob_fn, + ) + chunks: list[torch.Tensor] = [] + observed_ranges: list[tuple[int, int]] = [] + for start in range(0, batch.input_ids.shape[0], batch_size): + stop = min(start + batch_size, batch.input_ids.shape[0]) + inputs = StatelessForwardInputs( + input_ids=batch.input_ids[start:stop], + attention_mask=batch.attention_mask[start:stop], + completion_mask=batch.active_mask[start:stop], + labels=batch.selected_token_ids[start:stop], + position_ids=( + None if batch.position_ids is None else batch.position_ids[start:stop] + ), + ) + result = executor.score(inputs) + if result.reference_logps is None: # pragma: no cover - guarded by config mode + raise RuntimeError("stateless scorer returned no selected logprobs") + chunks.append(result.reference_logps.detach().to(device="cpu")) + observed_ranges.append((start, stop)) + selected = torch.cat(chunks, dim=0) + return ( + RankScore( + rank=0, + world_size=1, + selected_logprobs=selected, + metadata={ + "device": "cpu", + "teacher_forcing": True, + "use_cache": False, + "optimizer_step": False, + "batch_ranges": observed_ranges, + }, + ), + ) + + +class CpuSmokeMaterializer: + """Materialize the exact single-process CPU surface used by smoke tests.""" + + runtime_kind = "cpu_smoke" + + @property + def implementation_fingerprint(self) -> str: + """Seal the adapter's concrete class and materialization entry point.""" + + return fingerprint_implementation( + type(self), + instance=self, + entrypoints=("materialize",), + ) + + def __init__( + self, + *, + requested_operator_backends: Optional[Mapping[str, str]] = None, + actual_operator_backends: Optional[Mapping[str, str]] = None, + ): + self.requested_operator_backends = dict(requested_operator_backends or {}) + self.actual_operator_backends = dict(actual_operator_backends or {}) + + def materialize( + self, + normalized: Mapping[str, Any], + descriptors: Mapping[str, KnobDescriptor], + ) -> AdapterMaterialization: + flat = _flatten(normalized) + applications = tuple( + self._application(path, value, descriptors[path]) for path, value in flat.items() + ) + batch_size = int(flat["batch.size"]) + requested_logp = str(flat["logp.backend"]) + operator_backends = self.requested_operator_backends or { + "rollout": requested_logp, + "training": requested_logp, + } + return AdapterMaterialization( + applications=applications, + binding=RuntimeBinding( + batch_size=batch_size, + side_configs={ + "rollout": { + "device": "cpu", + "dtype": "float32", + "enable_prefix_caching": False, + "enforce_eager": True, + }, + "training": { + "device": "cpu", + "dtype": "float32", + "attention_backend": "eager", + "sharding": "unsharded", + }, + }, + topology={ + "rollout": { + "world_size": 1, + "tensor_parallel_size": 1, + "context_parallel_size": 1, + }, + "training": {"world_size": 1, "sharding": "unsharded"}, + }, + scorer={ + "mode": "reference", + "use_cache": False, + "attention_backend": "eager", + "output_dtype": "float32", + }, + operator_backends=operator_backends, + runtime_kind=self.runtime_kind, + ), + ) + + def _application( + self, + path: str, + requested: Any, + descriptor: KnobDescriptor, + ) -> KnobApplication: + fixed_values = { + "rollout.tensor_parallel_size": 1, + "rollout.context_parallel_size": 1, + "rollout.dtype": "float32", + "rollout.enable_prefix_caching": False, + "rollout.enforce_eager": True, + "training.attention_backend": "eager", + "training.compute_dtype": "float32", + "training.sharding": "unsharded", + } + if path == "batch.size": + return _application( + descriptor, + requested, + requested, + requested, + MaterializationStatus.APPLIED, + "canonical batch is partitioned at scorer invocation", + ) + if path == "logp.backend": + requested_backends = self.requested_operator_backends or { + "rollout": requested, + "training": requested, + } + if requested_backends.get("rollout") != requested: + return _application( + descriptor, + requested, + requested_backends, + None, + MaterializationStatus.ERROR, + "rollout operator conflicts with public logp.backend", + ) + actual_backends = { + "rollout": self.actual_operator_backends.get("rollout"), + "training": self.actual_operator_backends.get("training"), + } + if None in actual_backends.values(): + return _application( + descriptor, + requested, + requested_backends, + None, + MaterializationStatus.UNOBSERVABLE, + "operator resolution trace has not been supplied", + ) + status = ( + MaterializationStatus.APPLIED + if actual_backends == requested_backends + else MaterializationStatus.FALLBACK + ) + return _application( + descriptor, + requested, + requested_backends, + actual_backends, + status, + "concrete CPU backends were read from exact resolution traces", + ) + + actual = fixed_values[path] + status = ( + MaterializationStatus.APPLIED + if requested == actual + else MaterializationStatus.UNSUPPORTED + ) + reason = ( + "read back from the single-process CPU scorer" + if status is MaterializationStatus.APPLIED + else f"CPU smoke supports only {path}={actual!r}" + ) + return _application(descriptor, requested, requested, actual, status, reason) + + +def run_cpu_experiment( + config: ExperimentConfig, + *, + output_root: str | Path, + allow_smoke_operators: bool = False, + timeout_seconds: float = 30.0, + resume: bool = True, +) -> dict[str, Any]: + """Run every planned case through the explicit CPU smoke adapter.""" + + scenario_device = str(config.definition.scenario.get("device", "")).strip().lower() + if scenario_device != "cpu": + raise ValueError("the CPU runtime requires scenario.device='cpu'") + plan = build_execution_plan(config) + + store = ArtifactStore(output_root) + experiment_dir = store.initialize_experiment( + config.definition.experiment_id, + experiment=plan.experiment, + plan=plan.rows(), + ) + batch = canonical_cpu_batch(config) + runs = [ + run_cpu_case( + store, + entry.case, + batch, + entry.operators, + allow_smoke_operators=allow_smoke_operators, + strict=config.definition.strict_fallback, + timeout_seconds=timeout_seconds, + resume=resume, + ) + for entry in plan.entries + ] + cases = [ + { + "case_id": run.case_id, + "attempt_id": run.attempt_id, + "status": str(run.summary["status"]), + "rollout_backend": run.summary["rollout_backend"], + "training_backend": run.summary["training_backend"], + "mismatch_count": run.summary.get("mismatch_count"), + "worst_token_index": run.summary.get("worst_token_index"), + "resumed": run.resumed, + "attempt_dir": str(run.attempt_dir), + } + for run in runs + ] + return { + "schema_version": "cross_config.cli_summary.v1", + "status": "pass" if all(item["status"] == "pass" for item in cases) else "fail", + "experiment_id": config.definition.experiment_id, + "scenario_id": config.definition.scenario_id, + "runtime": "cpu-smoke", + "artifact_dir": str(experiment_dir), + "cases": cases, + } + + +def run_cpu_case( + store: ArtifactStore, + case: ExperimentCase, + batch: CanonicalScoringBatch, + selection: OperatorSelection, + *, + allow_smoke_operators: bool, + strict: bool, + timeout_seconds: float, + resume: bool, +) -> PairedRunResult: + """Execute one already-bound CPU case with case-local operator state.""" + + catalog = SemanticOperatorCatalog(kernel_registry.semantic.backend_descriptors()) + if allow_smoke_operators: + from rl_engine.alignment.testing.smoke_ops import register_smoke_operators + + register_smoke_operators(catalog, allow_smoke_operators=True) + bridge = OperatorBridge( + catalog, + policy=OperatorResolutionPolicy( + strict=strict, + allow_test_backends=allow_smoke_operators, + ), + ) + override = OperatorOverride( + semantic_op="selected_logprob", + rollout_backend=selection.rollout_backend, + training_backend=selection.training_backend, + ) + topologies: dict[str, Mapping[str, Any]] = { + "rollout": { + "world_size": 1, + "tensor_parallel_size": 1, + "context_parallel_size": 1, + }, + "training": {"world_size": 1, "sharding": "unsharded"}, + } + rollout_dtype = str(case.requested["rollout"]["dtype"]) + training_dtype = str(case.requested["training"]["compute_dtype"]) + requirements = { + "rollout": OperatorRequirements( + device="cpu", + dtype=rollout_dtype, + topology=topologies["rollout"], + alignment_properties={"deterministic": True}, + ), + "training": OperatorRequirements( + device="cpu", + dtype=training_dtype, + topology=topologies["training"], + alignment_properties={"deterministic": True}, + ), + } + resolved = bridge.resolve_override(override, requirements=requirements, strict=strict) + options = { + target: _factory_options( + selection.backend_for(target), + selection.options_for(target), + allow_smoke_operators=allow_smoke_operators, + ) + for target in ("rollout", "training") + } + instances = { + target: bridge.instantiate( + resolved, + target=target, # type: ignore[arg-type] + factory_kwargs=options[target], + ) + for target in ("rollout", "training") + } + provenance = { + target: bridge.instance_provenance( + resolved, + target=target, # type: ignore[arg-type] + instance=instances[target], + ) + for target in ("rollout", "training") + } + actual_backends = {target: provenance[target].backend_id for target in provenance} + materialization = RuntimeTools().materialize( + case, + CpuSmokeMaterializer( + requested_operator_backends={ + "rollout": selection.rollout_backend, + "training": selection.training_backend, + }, + actual_operator_backends=actual_backends, + ), + ) + RuntimeTools.require_executable(materialization, strict=strict) + + minimum_token_id = min( + int(batch.input_ids.min().item()), + int(batch.selected_token_ids.min().item()), + ) + if minimum_token_id < 0: + raise ValueError("CPU smoke token IDs must be non-negative") + vocab_size = ( + max( + int(batch.input_ids.max().item()), + int(batch.selected_token_ids.max().item()), + ) + + 17 + ) + scorers = { + "rollout": CpuStatelessScorer( + SyntheticCpuCausalLM(vocab_size), + _scorer_spec( + ScoreSide.ROLLOUT, + rollout_dtype, + selection.rollout_backend, + topologies["rollout"], + case, + ), + ), + "training": CpuStatelessScorer( + SyntheticCpuCausalLM(vocab_size), + _scorer_spec( + ScoreSide.TRAINING, + training_dtype, + selection.training_backend, + topologies["training"], + case, + ), + ), + } + return PairedRunner(store, timeout_seconds=timeout_seconds).run( + case, + materialization, + batch, + scorers["rollout"], + scorers["training"], + resolved, + instances, + provenance, + operator_factory_options=options, + strict=strict, + timeout_seconds=timeout_seconds, + resume=resume, + ) + + +def canonical_cpu_batch(config: ExperimentConfig) -> CanonicalScoringBatch: + """Build the immutable CPU tensors frozen by an experiment identity.""" + + identity = config.definition.identity + position_ids = ( + torch.tensor(identity.position_ids, dtype=torch.long, device="cpu") + if identity.position_ids + else None + ) + return CanonicalScoringBatch( + identity=identity, + input_ids=torch.tensor(identity.token_ids, dtype=torch.long, device="cpu"), + selected_token_ids=torch.tensor( + identity.selected_token_ids, + dtype=torch.long, + device="cpu", + ), + active_mask=torch.tensor(identity.active_mask, dtype=torch.bool, device="cpu"), + attention_mask=torch.tensor( + identity.attention_mask, + dtype=torch.bool, + device="cpu", + ), + position_ids=position_ids, + metadata={"source": "named_json", "device": "cpu"}, + ) + + +def _factory_options( + backend_id: str, + configured: Mapping[str, Any], + *, + allow_smoke_operators: bool, +) -> dict[str, Any]: + options = dict(configured) + if backend_id == "smoke_only.logp_offset": + if not allow_smoke_operators: + raise PermissionError("smoke offset requires explicit test authorization") + options["allow_smoke_operators"] = True + return options + + +def _scorer_spec( + side: ScoreSide, + dtype: str, + backend_id: str, + topology: Mapping[str, Any], + case: ExperimentCase, +) -> ScorerSpec: + identity = case.identity + return ScorerSpec( + side=side, + backend_id="cpu_stateless_teacher_forcing", + dtype=dtype, + device="cpu", + world_size=1, + topology=topology, + construction_options={ + "checkpoint_id": identity.checkpoint_id, + "model_version": identity.model_version, + "pre_update_state": identity.pre_update_state, + "teacher_forcing": True, + "use_cache": False, + }, + operator_overrides={"selected_logprob": backend_id}, + ) + + +def _application( + descriptor: KnobDescriptor, + requested: Any, + materialized: Any, + actual: Any, + status: MaterializationStatus, + reason: str, +) -> KnobApplication: + return KnobApplication( + path=descriptor.path, + requested=requested, + materialized=materialized, + actual=actual, + lifecycle=descriptor.lifecycle, + status=status, + critical=descriptor.critical, + evidence={"reason": reason}, + ) + + +def _flatten(value: Mapping[str, Any], prefix: str = "") -> dict[str, Any]: + result: dict[str, Any] = {} + for key, child in value.items(): + path = f"{prefix}.{key}" if prefix else key + if isinstance(child, Mapping): + result.update(_flatten(child, path)) + else: + result[path] = child + return result + + +def _require_module_on_cpu(model: torch.nn.Module) -> None: + tensors = tuple(model.parameters()) + tuple(model.buffers()) + if any(tensor.device.type != "cpu" for tensor in tensors): + raise ValueError("CPU smoke models must remain on CPU") + + +def _require_module_float_dtype(model: torch.nn.Module, expected: torch.dtype) -> None: + tensors = tuple(model.parameters()) + tuple(model.buffers()) + mismatched = sorted( + { + str(tensor.dtype).replace("torch.", "") + for tensor in tensors + if tensor.is_floating_point() and tensor.dtype is not expected + } + ) + if mismatched: + raise ValueError( + f"CPU smoke model floating dtype must be {expected}; observed {mismatched}" + ) + + +def _device_type(value: str) -> str: + return value.split(":", 1)[0].strip().lower() + + +def _torch_dtype(value: str) -> torch.dtype: + normalized = value.strip().lower().replace("torch.", "") + aliases = {"fp32": "float32", "bf16": "bfloat16", "fp16": "float16"} + normalized = aliases.get(normalized, normalized) + try: + return { + "float32": torch.float32, + "bfloat16": torch.bfloat16, + "float16": torch.float16, + }[normalized] + except KeyError as exc: + raise ValueError(f"unsupported scorer dtype {value!r}") from exc + + +__all__ = [ + "CpuSmokeMaterializer", + "CpuStatelessScorer", + "SyntheticCpuCausalLM", + "canonical_cpu_batch", + "run_cpu_case", + "run_cpu_experiment", +] diff --git a/rl_engine/alignment/testing/smoke_ops/SMOKE_OPERATORS.md b/rl_engine/alignment/testing/smoke_ops/SMOKE_OPERATORS.md new file mode 100644 index 00000000..0ff9e6ea --- /dev/null +++ b/rl_engine/alignment/testing/smoke_ops/SMOKE_OPERATORS.md @@ -0,0 +1,30 @@ +# Cross-configuration smoke-only operators + +These files are temporary test scaffolding. They validate operator selection, +strict resolution, active-token scoring, and provenance; they do not establish +production numerical alignment. + +| File | Backend | Purpose | Replacement owner / issue | +| --- | --- | --- | --- | +| `smoke_only_logp_reference.py` | `smoke_only.logp_reference` | CPU PyTorch `log_softmax` plus gather reference for rollout/training injection tests. | Production selected-logprob operator workstream; roadmap issue #83 / WS1 contract issue #108. | +| `smoke_only_logp_offset.py` | `smoke_only.logp_offset` | Adds an explicit deterministic active-token offset so comparator mismatch detection can be tested. | Test-only fault injection; no production replacement should preserve the offset. | +| `__init__.py` | registration boundary | Keeps registration disabled by default and requires `allow_smoke_operators=True`. | Remove with both smoke implementations. | + +Removal trigger: delete this package once equivalent production RL-Kernel +selected-logprob operators are integrated and the same framework tests pass using +those production backends on both rollout and training sides. + +Exact deletion steps: + +1. Change `tests/test_cross_config_runtime.py` to exercise the production backend + IDs while preserving disabled/unavailable, capability, paired-output, and + provenance coverage. +2. Remove the `smoke_operator` test marker if no other temporary smoke operator + tests use it. +3. Delete `rl_engine/alignment/testing/smoke_ops/` and remove its exports from + `rl_engine/alignment/testing/__init__.py`. +4. Search the repository for `smoke_only.`, `allow_smoke_operators`, and + `RL_KERNEL_ALLOW_SMOKE_OPS`; remove configuration and documentation references + that no longer describe an active test boundary. +5. Run the cross-configuration contract, runtime, runner, and production-backend + tests before merging the deletion. diff --git a/rl_engine/alignment/testing/smoke_ops/__init__.py b/rl_engine/alignment/testing/smoke_ops/__init__.py new file mode 100644 index 00000000..5b6296ae --- /dev/null +++ b/rl_engine/alignment/testing/smoke_ops/__init__.py @@ -0,0 +1,75 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Opt-in registration for temporary Cross-configuration alignment smoke-only operators.""" + +from __future__ import annotations + +from typing import Any + +from rl_engine.kernels.semantic_registry import OperatorBackendDescriptor, SemanticOperatorCatalog + +SMOKE_ONLY_LOGP_REFERENCE_BACKEND_ID = "smoke_only.logp_reference" +SMOKE_ONLY_LOGP_OFFSET_BACKEND_ID = "smoke_only.logp_offset" + + +def smoke_operator_descriptors() -> tuple[OperatorBackendDescriptor, ...]: + """Build smoke descriptors lazily without registering them globally.""" + + from .smoke_only_logp_offset import SMOKE_ONLY_LOGP_OFFSET_DESCRIPTOR + from .smoke_only_logp_reference import SMOKE_ONLY_LOGP_REFERENCE_DESCRIPTOR + + return ( + SMOKE_ONLY_LOGP_REFERENCE_DESCRIPTOR, + SMOKE_ONLY_LOGP_OFFSET_DESCRIPTOR, + ) + + +def register_smoke_operators( + catalog: SemanticOperatorCatalog, + *, + allow_smoke_operators: bool = False, + replace: bool = False, +) -> tuple[OperatorBackendDescriptor, ...]: + """Register every smoke backend after an explicit per-call opt-in. + + Importing this package never mutates a catalog. Resolution independently + requires an ``OperatorResolutionPolicy`` that allows test backends; this + registration guard is the first fail-closed boundary. + """ + + if allow_smoke_operators is not True: + raise PermissionError( + "smoke operator registration requires explicit " "allow_smoke_operators=True" + ) + if not isinstance(catalog, SemanticOperatorCatalog): + raise TypeError("catalog must be a SemanticOperatorCatalog") + + descriptors = smoke_operator_descriptors() + for descriptor in descriptors: + catalog.register_backend(descriptor, replace=replace) + return descriptors + + +def __getattr__(name: str) -> Any: + """Lazily expose implementation classes without default torch imports.""" + + if name == "SmokeOnlyLogpReference": + from .smoke_only_logp_reference import SmokeOnlyLogpReference + + return SmokeOnlyLogpReference + if name == "SmokeOnlyLogpOffset": + from .smoke_only_logp_offset import SmokeOnlyLogpOffset + + return SmokeOnlyLogpOffset + raise AttributeError(name) + + +__all__ = [ + "SMOKE_ONLY_LOGP_OFFSET_BACKEND_ID", + "SMOKE_ONLY_LOGP_REFERENCE_BACKEND_ID", + "SmokeOnlyLogpOffset", + "SmokeOnlyLogpReference", + "register_smoke_operators", + "smoke_operator_descriptors", +] diff --git a/rl_engine/alignment/testing/smoke_ops/smoke_only_logp_offset.py b/rl_engine/alignment/testing/smoke_ops/smoke_only_logp_offset.py new file mode 100644 index 00000000..2a0c6554 --- /dev/null +++ b/rl_engine/alignment/testing/smoke_ops/smoke_only_logp_offset.py @@ -0,0 +1,93 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""TEMPORARY TEST SCAFFOLD - NOT A PRODUCTION RL-KERNEL OPERATOR""" + +from __future__ import annotations + +import math +from typing import Optional + +import torch + +from rl_engine.kernels.semantic_registry import ( + OperatorBackendDescriptor, + OperatorFallbackPolicy, + OperatorLifecycle, +) + +from . import SMOKE_ONLY_LOGP_OFFSET_BACKEND_ID +from .smoke_only_logp_reference import SmokeOnlyLogpReference + + +class SmokeOnlyLogpOffset(SmokeOnlyLogpReference): + """CPU reference plus a deterministic test-only active-token offset. + + Inputs and output follow :class:`SmokeOnlyLogpReference`. ``offset`` is zero + by default. A non-zero value requires the constructor's explicit + ``allow_smoke_operators=True`` guard. The cross-configuration bridge masks + inactive output positions after invocation, so drift applies only to active + selected tokens. + """ + + def __init__( + self, + offset: float = 0.0, + *, + allow_smoke_operators: bool = False, + ) -> None: + normalized_offset = float(offset) + if not math.isfinite(normalized_offset): + raise ValueError("offset must be finite") + if normalized_offset != 0.0 and allow_smoke_operators is not True: + raise PermissionError( + "a non-zero smoke offset requires explicit " "allow_smoke_operators=True" + ) + self.offset = normalized_offset + + def apply_fp32( + self, + logits: torch.Tensor, + token_ids: torch.Tensor, + active_mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + selected = super().apply_fp32(logits, token_ids, active_mask=active_mask) + if self.offset == 0.0: + return selected + if active_mask is None: + return selected + self.offset + mask = active_mask.to(device=selected.device, dtype=torch.bool) + return selected + mask.to(dtype=selected.dtype) * self.offset + + +SMOKE_ONLY_LOGP_OFFSET_DESCRIPTOR = OperatorBackendDescriptor( + semantic_op="selected_logprob", + backend_id=SMOKE_ONLY_LOGP_OFFSET_BACKEND_ID, + supported_targets=frozenset({"rollout", "training"}), + supported_devices=frozenset({"cpu"}), + supported_dtypes=frozenset({"bfloat16", "float16", "float32"}), + supported_topologies={ + "world_size": (1,), + "tensor_parallel_size": (1,), + "context_parallel_size": (1,), + "sharding": ("unsharded",), + }, + determinism_or_alignment_properties={ + "algorithm": "pytorch.log_softmax_gather_plus_test_offset", + "batch_invariant": True, + "deterministic": True, + "strict_observable": True, + "test_offset_configurable": True, + }, + lifecycle=OperatorLifecycle.ENGINE_CONSTRUCTION, + implementation_class_or_factory=SmokeOnlyLogpOffset, + fallback_policy=OperatorFallbackPolicy.ERROR, + version_or_build_fingerprint="cross-config-smoke-only-logp-offset-v1", + is_smoke_only=True, +) + + +__all__ = [ + "SMOKE_ONLY_LOGP_OFFSET_DESCRIPTOR", + "SmokeOnlyLogpOffset", +] diff --git a/rl_engine/alignment/testing/smoke_ops/smoke_only_logp_reference.py b/rl_engine/alignment/testing/smoke_ops/smoke_only_logp_reference.py new file mode 100644 index 00000000..342794a7 --- /dev/null +++ b/rl_engine/alignment/testing/smoke_ops/smoke_only_logp_reference.py @@ -0,0 +1,108 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""TEMPORARY TEST SCAFFOLD - NOT A PRODUCTION RL-KERNEL OPERATOR""" + +from __future__ import annotations + +from typing import Optional + +import torch + +from rl_engine.kernels.semantic_registry import ( + OperatorBackendDescriptor, + OperatorFallbackPolicy, + OperatorLifecycle, +) + +from . import SMOKE_ONLY_LOGP_REFERENCE_BACKEND_ID + + +class SmokeOnlyLogpReference: + """CPU selected-logprob reference used only to test operator plumbing. + + The semantic inputs are logits shaped ``[..., vocabulary]`` and selected + token IDs shaped ``[...]``. The result is one float32 log probability per + selected token. When supplied, ``active_mask`` has the token-ID shape and + inactive output positions are exactly zero. The cross-configuration bridge + applies the same masking rule when invoking the two-argument interface. + """ + + op_class = "logprob" + + def __call__( + self, + logits: torch.Tensor, + token_ids: torch.Tensor, + active_mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + return self.apply_fp32(logits, token_ids, active_mask=active_mask) + + def apply_fp32( + self, + logits: torch.Tensor, + token_ids: torch.Tensor, + active_mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Compute CPU log-softmax/gather output with optional active masking.""" + + _validate_inputs(logits, token_ids, active_mask) + selected_ids = token_ids.to(device=logits.device, dtype=torch.long) + mask = None + if active_mask is not None: + mask = active_mask.to(device=logits.device, dtype=torch.bool) + selected_ids = selected_ids.masked_fill(~mask, 0) + + log_probs = torch.log_softmax(logits.float(), dim=-1) + selected = torch.gather(log_probs, dim=-1, index=selected_ids.unsqueeze(-1)).squeeze(-1) + if mask is not None: + selected = selected.masked_fill(~mask, 0.0) + return selected + + +def _validate_inputs( + logits: torch.Tensor, + token_ids: torch.Tensor, + active_mask: Optional[torch.Tensor], +) -> None: + if logits.device.type != "cpu": + raise ValueError("smoke-only logprob operators support CPU tensors only") + if logits.shape[:-1] != token_ids.shape: + raise ValueError( + f"logits leading shape {tuple(logits.shape[:-1])} must match " + f"token_ids shape {tuple(token_ids.shape)}" + ) + if active_mask is not None and active_mask.shape != token_ids.shape: + raise ValueError("active_mask shape must match token_ids shape") + + +SMOKE_ONLY_LOGP_REFERENCE_DESCRIPTOR = OperatorBackendDescriptor( + semantic_op="selected_logprob", + backend_id=SMOKE_ONLY_LOGP_REFERENCE_BACKEND_ID, + supported_targets=frozenset({"rollout", "training"}), + supported_devices=frozenset({"cpu"}), + supported_dtypes=frozenset({"bfloat16", "float16", "float32"}), + supported_topologies={ + "world_size": (1,), + "tensor_parallel_size": (1,), + "context_parallel_size": (1,), + "sharding": ("unsharded",), + }, + determinism_or_alignment_properties={ + "algorithm": "pytorch.log_softmax_gather", + "batch_invariant": True, + "deterministic": True, + "strict_observable": True, + }, + lifecycle=OperatorLifecycle.ENGINE_CONSTRUCTION, + implementation_class_or_factory=SmokeOnlyLogpReference, + fallback_policy=OperatorFallbackPolicy.ERROR, + version_or_build_fingerprint="cross-config-smoke-only-logp-reference-v1", + is_smoke_only=True, +) + + +__all__ = [ + "SMOKE_ONLY_LOGP_REFERENCE_DESCRIPTOR", + "SmokeOnlyLogpReference", +] From ff4911dca1d56a79bfa434ca4fd73d875e4c8527 Mon Sep 17 00:00:00 2001 From: CyberSecurityErial <2710555967@qq.com> Date: Sun, 19 Jul 2026 08:35:20 +0800 Subject: [PATCH 7/8] test(alignment): add focused cross-configuration coverage --- .github/workflows/ci.yml | 8 + pyproject.toml | 5 + tests/test_cross_config_cli.py | 145 ++++++ tests/test_cross_config_contract.py | 409 +++++++++++++++ tests/test_cross_config_runner.py | 646 +++++++++++++++++++++++ tests/test_cross_config_runtime.py | 759 ++++++++++++++++++++++++++++ tests/test_stateless_executor.py | 84 +++ tests/test_tolerance_contract.py | 51 +- 8 files changed, 2106 insertions(+), 1 deletion(-) create mode 100644 tests/test_cross_config_cli.py create mode 100644 tests/test_cross_config_contract.py create mode 100644 tests/test_cross_config_runner.py create mode 100644 tests/test_cross_config_runtime.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 92cd0433..e9e9c91d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,6 +68,14 @@ jobs: python -m pytest rl_engine/tests/test_dispatch.py -v PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python -m pytest tests/test_attention_correctness.py -q -rs + - name: Run Cross-Configuration Contract Tests (CPU-safe) + run: | + PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python -m pytest -q \ + tests/test_cross_config_*.py \ + tests/test_stateless_executor.py \ + tests/test_tolerance_contract.py \ + tests/test_kernel_registry.py + - name: Run Attention Ground-Truth Tests (CPU-safe) run: | python -m pytest tests/test_attention.py -v -k "not large and not gpu" diff --git a/pyproject.toml b/pyproject.toml index 9c8300ce..d4ed5c57 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,3 +43,8 @@ ignore = [] [tool.mypy] ignore_missing_imports = true follow_imports = "silent" + +[tool.pytest.ini_options] +markers = [ + "smoke_operator: temporary smoke-only operator plumbing tests", +] diff --git a/tests/test_cross_config_cli.py b/tests/test_cross_config_cli.py new file mode 100644 index 00000000..8193f5df --- /dev/null +++ b/tests/test_cross_config_cli.py @@ -0,0 +1,145 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest +import torch + +import rl_engine.alignment.cross_config.__main__ as cli_main +from rl_engine.alignment.cross_config.artifacts import ArtifactStore + +_REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +_EXAMPLES = _REPOSITORY_ROOT / "examples" +_CPU_RUNTIME_MODULE = "rl_engine.alignment.testing.cpu_cross_config" + + +def _summary(captured: str) -> dict: + summaries = [] + for line in captured.splitlines(): + try: + value = json.loads(line) + except json.JSONDecodeError: + continue + if value.get("schema_version") == "cross_config.cli_summary.v1": + summaries.append(value) + assert len(summaries) == 1 + return summaries[0] + + +def test_run_uses_only_cpu_and_resumes_when_cuda_is_available(tmp_path, capsys, monkeypatch): + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + config_path = _EXAMPLES / "cross_config_s0_cpu_smoke.json" + plan_argv = [ + "plan", + str(config_path), + "--output-root", + str(tmp_path), + ] + argv = [ + "run", + str(config_path), + "--runtime", + "cpu-smoke", + "--allow-smoke-operators", + "--output-root", + str(tmp_path), + "--timeout-seconds", + "10", + ] + + assert cli_main.main(plan_argv) == 0 + planned = _summary(capsys.readouterr().out) + experiment_path = Path(planned["artifact_dir"]) / "experiment.json" + plan_path = Path(planned["artifact_dir"]) / "plan.jsonl" + planned_experiment = experiment_path.read_bytes() + planned_cases = plan_path.read_bytes() + stored_config = json.loads(planned_experiment) + stored_row = json.loads(planned_cases) + assert stored_config["schema_version"] == "cross_config.experiment_config.v1" + assert stored_row["schema_version"] == "cross_config.execution_plan_entry.v1" + assert stored_row["case"]["execution_binding"]["operators"] == stored_row["operators"] + + assert cli_main.main(argv) == 0 + captured = capsys.readouterr() + first = _summary(captured.out) + assert first["status"] == "pass" + assert first["runtime"] == "cpu-smoke" + assert "actual backends rollout=smoke_only.logp_reference" in captured.err + assert "training=smoke_only.logp_reference" in captured.err + assert "worst sample/token=(0, 3)" in captured.err + assert first["cases"] + assert all(case["status"] == "pass" for case in first["cases"]) + assert all(case["resumed"] is False for case in first["cases"]) + assert experiment_path.read_bytes() == planned_experiment + assert plan_path.read_bytes() == planned_cases + + store = ArtifactStore(tmp_path) + for case in first["cases"]: + attempt_dir = Path(case["attempt_dir"]) + assert (attempt_dir / "COMPLETE").is_file() + actual = json.loads((attempt_dir / "actual.json").read_text(encoding="utf-8")) + assert actual["environment"]["execution_devices"] == { + "rollout": "cpu", + "training": "cpu", + } + for name in ("score_rollout.pt", "score_training.pt"): + bundle = store.load_tensor_bundle(attempt_dir / name) + assert bundle["tensors"]["selected_logprobs"].device.type == "cpu" + + assert cli_main.main(argv) == 0 + resumed = _summary(capsys.readouterr().out) + assert resumed["status"] == "pass" + assert [case["attempt_id"] for case in resumed["cases"]] == [ + case["attempt_id"] for case in first["cases"] + ] + assert all(case["resumed"] is True for case in resumed["cases"]) + + +@pytest.mark.parametrize( + ("filename", "expected_cases"), + [ + ("cross_config_s1_distributed_smoke.json", 5), + ("cross_config_s2_vllm_tp_vs_fsdp.json", 10), + ("cross_config_s3_qwen3_8b_tp4_cp4_bf16.json", 11), + ], +) +def test_plan_records_named_scenarios_without_loading_a_runtime( + tmp_path, + capsys, + monkeypatch, + filename, + expected_cases, +): + monkeypatch.delitem(sys.modules, _CPU_RUNTIME_MODULE, raising=False) + + def runtime_must_not_run(*args, **kwargs): + raise AssertionError(f"plan unexpectedly invoked the CPU runtime: {args!r}, {kwargs!r}") + + monkeypatch.setattr(cli_main, "_run", runtime_must_not_run) + assert ( + cli_main.main( + [ + "plan", + str(_EXAMPLES / filename), + "--output-root", + str(tmp_path), + ] + ) + == 0 + ) + + summary = _summary(capsys.readouterr().out) + artifact_dir = Path(summary["artifact_dir"]) + assert summary["status"] == "planned" + assert summary["planned_case_count"] == expected_cases + assert (artifact_dir / "experiment.json").is_file() + assert len((artifact_dir / "plan.jsonl").read_text(encoding="utf-8").splitlines()) == ( + expected_cases + ) + assert not list(artifact_dir.glob("cases/*/*")) + assert _CPU_RUNTIME_MODULE not in sys.modules diff --git a/tests/test_cross_config_contract.py b/tests/test_cross_config_contract.py new file mode 100644 index 00000000..74809756 --- /dev/null +++ b/tests/test_cross_config_contract.py @@ -0,0 +1,409 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +from __future__ import annotations + +import json +from collections.abc import Mapping +from dataclasses import replace +from pathlib import Path +from typing import Any + +import pytest +import torch + +from rl_engine.alignment.cross_config.comparison import ( + compare_score_artifacts, + recompute_mismatch_mask, +) +from rl_engine.alignment.cross_config.config import ( + CONFIG_SCHEMA_VERSION, + bind_operator_selection, + load_config, +) +from rl_engine.alignment.cross_config.planner import Planner, PlanningError +from rl_engine.alignment.cross_config.schema import ( + AlignmentStatus, + ExperimentDefinition, + InterventionSpec, + PlanningStrategy, + RuntimeProvenance, + ScoreArtifact, + ScorerSpec, + ScoreSide, + SemanticIdentitySpec, +) +from rl_engine.kernels.gtest.tolerance import resolve_logprob_threshold + + +def _identity( + *, + checkpoint_id: str = "tiny-checkpoint", + tokenizer_policy: str = "tokenizer-v1:right-padding", + active_mask: tuple[tuple[bool, ...], ...] = ((True, False, True),), +) -> SemanticIdentitySpec: + return SemanticIdentitySpec( + checkpoint_id=checkpoint_id, + model_version="weights-v7", + tokenizer_id="tiny-tokenizer", + tokenizer_policy=tokenizer_policy, + token_ids=((11, 12, 13),), + selected_token_ids=((12, 13, 14),), + active_mask=active_mask, + attention_mask=((True, True, True),), + position_ids=((0, 1, 2),), + pre_update_state="state-before-step-9", + cache_metadata={"use_cache": False}, + packing_metadata={"packed": False}, + ) + + +def _score( + side: ScoreSide, + values: torch.Tensor, + *, + identity: SemanticIdentitySpec | None = None, + active_mask: torch.Tensor | None = None, +) -> ScoreArtifact: + identity = identity or _identity() + backend = f"test.{side.value}.selected_logprob" + return ScoreArtifact( + case_id="case-001", + attempt_id="attempt-001", + side=side, + identity=identity, + scorer=ScorerSpec( + side=side, + backend_id=f"{side.value}-scorer", + dtype="float32", + operator_overrides={"selected_logprob": backend}, + ), + selected_logprobs=values, + active_mask=( + active_mask + if active_mask is not None + else torch.tensor(identity.active_mask, dtype=torch.bool) + ), + provenance=RuntimeProvenance( + requested={"logp": {"backend": backend}}, + normalized={"logp": {"backend": backend}}, + materialized={"logp": {"backend": backend}}, + actual={"logp": {"backend": backend}}, + implementation_fingerprint=f"{side.value}-implementation-v1", + ), + ) + + +def _tensor_from_payload(payload: Mapping[str, Any]) -> torch.Tensor: + return torch.tensor(payload["values"], dtype=getattr(torch, str(payload["dtype"]))).reshape( + payload["shape"] + ) + + +def _baseline() -> dict[str, Any]: + return { + "batch": {"size": 8}, + "rollout": { + "tensor_parallel_size": 1, + "context_parallel_size": 1, + "dtype": "float32", + "enable_prefix_caching": False, + "enforce_eager": True, + }, + "training": { + "sharding": "unsharded", + "attention_backend": "eager", + "compute_dtype": "float32", + }, + "logp": {"backend": "rlkernel.reference_logp"}, + } + + +def _definition( + *, + strategy: PlanningStrategy = PlanningStrategy.ONE_AT_A_TIME, + pairwise_paths: tuple[tuple[str, str], ...] = (), +) -> ExperimentDefinition: + return ExperimentDefinition( + experiment_id="planner-test", + scenario_id="cpu-contract", + scenario={"model": "synthetic", "device": "cpu"}, + identity=_identity(), + baseline=_baseline(), + interventions=( + InterventionSpec("batch.size", (1, 4)), + InterventionSpec("rollout.dtype", ("bfloat16",)), + InterventionSpec("training.attention_backend", ("sdpa",)), + ), + strategy=strategy, + pairwise_paths=pairwise_paths, + ) + + +def _flatten(value: Mapping[str, Any], prefix: str = "") -> dict[str, Any]: + result: dict[str, Any] = {} + for key, child in value.items(): + path = f"{prefix}.{key}" if prefix else key + if isinstance(child, Mapping): + result.update(_flatten(child, path)) + else: + result[path] = child + return result + + +def _config() -> dict[str, Any]: + return { + "schema_version": CONFIG_SCHEMA_VERSION, + "experiment_id": "cpu-config-test", + "scenario_id": "cpu-smoke", + "contract_source": "ws1", + "contract_version": "current", + "strategy": "one_at_a_time", + "strict_fallback": True, + "identity": { + "checkpoint_id": "tiny", + "model_version": "weights-v1", + "tokenizer_policy": "synthetic-v1", + "token_ids": [[1, 2, 3]], + "selected_token_ids": [[0, 2, 3]], + "active_mask": [[False, True, True]], + "attention_mask": [[True, True, True]], + "pre_update_state": "iteration-0", + }, + "baseline": { + **_baseline(), + "batch": {"size": 1}, + }, + "interventions": [{"path": "batch.size", "values": [2]}], + "operators": { + "selected_logprob": { + "rollout": "rlkernel.reference_logp", + "training": { + "backend": "smoke_only.logp_offset", + "options": {"offset": 0.1}, + }, + } + }, + "scenario": {"device": "cpu", "workload": "tiny"}, + } + + +def _write_config(tmp_path: Path, value: Mapping[str, Any], name: str = "config.json") -> Path: + path = tmp_path / name + path.write_text(json.dumps(value), encoding="utf-8") + return path + + +def test_fixed_threshold_uses_only_active_tokens_and_is_reproducible_offline(): + threshold = resolve_logprob_threshold("float32") + rollout_values = torch.zeros((1, 3), dtype=torch.float32) + training_values = torch.tensor( + [[threshold * 2.0, 1_000.0, threshold * 0.5]], + dtype=torch.float32, + ) + + result = compare_score_artifacts( + _score(ScoreSide.ROLLOUT, rollout_values), + _score(ScoreSide.TRAINING, training_values), + ) + + assert result.status is AlignmentStatus.FAIL + assert result.active_token_count == 2 + assert result.mismatch_count == 1 + assert result.fixed_threshold == threshold + assert result.token_artifact is not None + assert result.token_artifact.mismatch_mask.tolist() == [[True, False, False]] + + payload = json.loads(json.dumps(result.token_artifact.to_dict())) + offline = recompute_mismatch_mask( + _tensor_from_payload(payload["rollout_logprobs"]), + _tensor_from_payload(payload["training_logprobs"]), + _tensor_from_payload(payload["active_mask"]), + float(payload["fixed_threshold"]), + ) + assert torch.equal(offline, result.token_artifact.mismatch_mask) + assert not recompute_mismatch_mask( + torch.zeros(1, dtype=torch.float64), + torch.tensor([threshold], dtype=torch.float64), + torch.tensor([True]), + threshold, + ).item() + + +def test_zero_tokens_identity_mismatch_and_invalid_scores_are_not_numerical_failures(): + empty_identity = _identity(active_mask=((False, False, False),)) + empty = compare_score_artifacts( + _score(ScoreSide.ROLLOUT, torch.zeros((1, 3)), identity=empty_identity), + _score(ScoreSide.TRAINING, torch.zeros((1, 3)), identity=empty_identity), + ) + assert empty.status is AlignmentStatus.ZERO_ACTIVE_TOKENS + assert empty.comparable is False + assert empty.passed is False + + identity = _identity() + changed_identity = replace(identity, tokenizer_policy="different-policy") + mismatched = compare_score_artifacts( + _score(ScoreSide.ROLLOUT, torch.zeros((1, 3)), identity=identity), + _score(ScoreSide.TRAINING, torch.ones((1, 3)), identity=changed_identity), + ) + assert mismatched.status is AlignmentStatus.INVALID_IDENTITY + assert "tokenizer_policy" in mismatched.identity_errors + + invalid = compare_score_artifacts( + _score(ScoreSide.ROLLOUT, torch.zeros((1, 3))), + _score(ScoreSide.TRAINING, torch.tensor([[float("nan"), 0.0, 0.0]])), + ) + assert invalid.status is AlignmentStatus.INVALID_ARTIFACT + assert invalid.comparable is False + + +def test_planner_emits_one_stable_baseline_and_one_change_per_oat_case(): + definition = _definition() + plan = Planner().plan(definition) + baseline = _flatten(plan.cases[0].requested) + + assert len(plan.cases) == 5 + assert sum(not case.changed_paths for case in plan.cases) == 1 + for case in plan.cases[1:]: + requested = _flatten(case.requested) + changed = {path for path, value in requested.items() if value != baseline[path]} + assert changed == set(case.changed_paths) + assert len(changed) == 1 + + reordered = replace( + definition, + experiment_id="same-plan-from-another-run", + baseline={ + "logp": {"backend": "reference"}, + "training": { + "compute_dtype": "fp32", + "attention_backend": "eager", + "sharding": "unsharded", + }, + "rollout": { + "enforce_eager": True, + "enable_prefix_caching": False, + "dtype": "fp32", + "context_parallel_size": 1, + "tensor_parallel_size": 1, + }, + "batch": {"size": 8}, + }, + ) + assert [case.case_id for case in plan.cases] == [ + case.case_id for case in Planner().plan(reordered).cases + ] + + +def test_pairwise_is_explicit_and_planning_errors_remain_structured(): + pairwise = _definition( + strategy=PlanningStrategy.PAIRWISE, + pairwise_paths=(("batch.size", "rollout.dtype"),), + ) + pairwise_cases = [ + case for case in Planner().plan(pairwise).cases if len(case.changed_paths) == 2 + ] + assert len(pairwise_cases) == 2 + assert all(case.changed_paths == ("batch.size", "rollout.dtype") for case in pairwise_cases) + + with pytest.raises(PlanningError) as not_enabled: + Planner().plan( + replace( + _definition(), + pairwise_paths=(("batch.size", "rollout.dtype"),), + ) + ) + assert {issue.code for issue in not_enabled.value.issues} == {"PAIRWISE_NOT_ENABLED"} + + invalid_requests = ( + ({"logp": {"tp_layout": "arbitrary"}}, "DERIVED_KNOB"), + ({"batch": {"size": True}}, "UNSUPPORTED_VALUE"), + ({"rollout": {"unknown": 1}}, "UNSUPPORTED_PATH"), + ) + for requested, expected_code in invalid_requests: + with pytest.raises(PlanningError) as invalid: + Planner().normalize_requested(requested) + assert invalid.value.issues[0].code == expected_code + + incomplete = replace( + _definition(), + baseline={key: value for key, value in _baseline().items() if key != "training"}, + ) + with pytest.raises(PlanningError) as missing: + Planner().plan(incomplete) + assert {issue.path for issue in missing.value.issues} == { + "training.attention_backend", + "training.compute_dtype", + "training.sharding", + } + assert all(issue.code == "MISSING_BASELINE_VALUE" for issue in missing.value.issues) + + +def test_versioned_config_loads_and_binds_target_specific_operators(tmp_path: Path): + loaded = load_config(_write_config(tmp_path, _config())) + base_case = loaded.plan().cases[0] + selection = loaded.operators_for(base_case) + bound = bind_operator_selection(base_case, selection) + + assert loaded.schema_version == CONFIG_SCHEMA_VERSION + assert loaded.definition.strategy is PlanningStrategy.ONE_AT_A_TIME + assert selection.rollout_backend == "rlkernel.reference_logp" + assert selection.training_backend == "smoke_only.logp_offset" + assert selection.training_options == {"offset": 0.1} + assert bound == bind_operator_selection(base_case, selection) + assert bound.case_id != base_case.case_id + assert bound.requested == base_case.requested + assert bound.execution_binding["operators"] == selection.to_dict() + + +def test_config_rejects_schema_escape_hatches_and_incomplete_operator_coverage( + tmp_path: Path, +): + wrong_schema = _config() + wrong_schema["schema_version"] = "cross_config.experiment_config.v999" + + unknown_key = _config() + unknown_key["strict_falback"] = True + + threshold_override = _config() + threshold_override["scenario"]["nested"] = {"threshold": 999.0} + + scenario_policy = _config() + scenario_policy["scenario"]["execution"] = "run" + + conflicting_axis = _config() + conflicting_axis["interventions"].append( + {"path": "logp.backend", "values": ["smoke_only.logp_offset"]} + ) + + incomplete_targets = _config() + del incomplete_targets["operators"]["selected_logprob"]["training"] + + invalid_configs = ( + (wrong_schema, "unsupported cross-configuration config schema"), + (unknown_key, "unknown config keys"), + (threshold_override, "fixed numerical-contract threshold"), + (scenario_policy, "scenario is metadata only"), + (conflicting_axis, "cannot be combined with logp.backend interventions"), + (incomplete_targets, "selected_logprob.training"), + ) + for index, (value, message) in enumerate(invalid_configs): + with pytest.raises(ValueError, match=message): + load_config(_write_config(tmp_path, value, f"invalid-{index}.json")) + + duplicate_key = tmp_path / "duplicate.json" + duplicate_key.write_text( + '{"schema_version":"cross_config.experiment_config.v1",' + '"schema_version":"cross_config.experiment_config.v1"}', + encoding="utf-8", + ) + with pytest.raises(ValueError, match="duplicate JSON key"): + load_config(duplicate_key) + + overflow = tmp_path / "overflow.json" + overflow.write_text( + json.dumps(_config()).replace('"size": 1', '"size": 1e400'), + encoding="utf-8", + ) + with pytest.raises(ValueError, match="non-finite JSON number"): + load_config(overflow) diff --git a/tests/test_cross_config_runner.py b/tests/test_cross_config_runner.py new file mode 100644 index 00000000..e37097c3 --- /dev/null +++ b/tests/test_cross_config_runner.py @@ -0,0 +1,646 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +from __future__ import annotations + +import json +import time +from pathlib import Path + +import pytest +import torch + +from rl_engine.alignment.cross_config.artifacts import ArtifactError, ArtifactStore +from rl_engine.alignment.cross_config.comparison import recompute_mismatch_mask +from rl_engine.alignment.cross_config.config import OperatorSelection, bind_operator_selection +from rl_engine.alignment.cross_config.operators import OperatorBridge, OperatorOverride +from rl_engine.alignment.cross_config.runner import ( + ChildScoringError, + PairedRunner, + RankCompletenessError, + RankScore, + ScoringTimeoutError, +) +from rl_engine.alignment.cross_config.runtime import RuntimeTools +from rl_engine.alignment.cross_config.schema import ( + CanonicalScoringBatch, + ExperimentCase, + ScorerSpec, + ScoreSide, + SemanticIdentitySpec, +) +from rl_engine.alignment.testing.cpu_cross_config import CpuSmokeMaterializer, run_cpu_case +from rl_engine.alignment.testing.smoke_ops import ( + SMOKE_ONLY_LOGP_OFFSET_BACKEND_ID, + SMOKE_ONLY_LOGP_REFERENCE_BACKEND_ID, +) +from rl_engine.alignment.testing.smoke_ops.smoke_only_logp_reference import SmokeOnlyLogpReference +from rl_engine.kernels.gtest.tolerance import resolve_logprob_threshold +from rl_engine.kernels.semantic_registry import OperatorRequirements + +_JSON_ARTIFACTS = ( + "requested.json", + "materialized.json", + "actual.json", + "identity.json", + "comparison.json", +) +_TENSOR_ARTIFACTS = ( + "score_rollout.pt", + "score_training.pt", + "token_diffs.pt", +) + + +class FixedRankScorer: + optimizer = None + model_state_fingerprint = "fixed-rank-scorer-state-v1" + + def __init__(self, spec: ScorerSpec, ranks): + self.spec = spec + self.ranks = tuple(ranks) + + def score(self, batch, *, batch_size, operator): + del batch_size, operator + return tuple( + RankScore( + rank=rank, + world_size=self.spec.world_size, + selected_logprobs=torch.zeros_like(batch.input_ids, dtype=torch.float32), + ) + for rank in self.ranks + ) + + +class FailingScorer(FixedRankScorer): + def score(self, batch, *, batch_size, operator): + del batch, batch_size, operator + raise RuntimeError("intentional scorer failure") + + +class SlowScorer(FixedRankScorer): + def score(self, batch, *, batch_size, operator): + time.sleep(2.0) + return super().score(batch, batch_size=batch_size, operator=operator) + + +def _identity() -> SemanticIdentitySpec: + token_ids = ( + (1, 2, 3, 4), + (2, 3, 4, 5), + (3, 4, 5, 6), + ) + selected = ( + (0, 2, 3, 4), + (0, 3, 4, 5), + (0, 4, 5, 6), + ) + active = tuple((False, True, True, True) for _ in token_ids) + attention = tuple((True, True, True, True) for _ in token_ids) + return SemanticIdentitySpec( + checkpoint_id="tiny-cpu-checkpoint", + model_version="weights-v1", + tokenizer_policy="synthetic-tokenizer-v1", + token_ids=token_ids, + selected_token_ids=selected, + active_mask=active, + attention_mask=attention, + pre_update_state="iteration-0", + ) + + +def _batch() -> CanonicalScoringBatch: + identity = _identity() + return CanonicalScoringBatch( + identity=identity, + input_ids=torch.tensor(identity.token_ids, device="cpu"), + selected_token_ids=torch.tensor(identity.selected_token_ids, device="cpu"), + active_mask=torch.tensor(identity.active_mask, device="cpu"), + attention_mask=torch.tensor(identity.attention_mask, device="cpu"), + metadata={"source": "runner-test", "device": "cpu"}, + ) + + +def _requested(*, backend: str = "rlkernel.reference_logp") -> dict[str, object]: + return { + "batch": {"size": 2}, + "rollout": { + "tensor_parallel_size": 1, + "context_parallel_size": 1, + "dtype": "float32", + "enable_prefix_caching": False, + "enforce_eager": True, + }, + "training": { + "attention_backend": "eager", + "compute_dtype": "float32", + "sharding": "unsharded", + }, + "logp": {"backend": backend}, + } + + +def _case( + *, + case_id: str = "case-runner", + backend: str = "rlkernel.reference_logp", +) -> ExperimentCase: + return ExperimentCase( + case_id=case_id, + experiment_id="runner-test", + scenario_id="S0", + identity=_identity(), + requested=_requested(backend=backend), + contract_fingerprint="contract-sha", + scenario_fingerprint="scenario-sha", + ) + + +def _topology(side: ScoreSide) -> dict[str, object]: + if side is ScoreSide.ROLLOUT: + return { + "world_size": 1, + "tensor_parallel_size": 1, + "context_parallel_size": 1, + } + return {"world_size": 1, "sharding": "unsharded"} + + +def _requirements(side: ScoreSide) -> OperatorRequirements: + return OperatorRequirements( + device="cpu", + dtype="float32", + topology=_topology(side), + alignment_properties={"deterministic": True}, + ) + + +def _operators(): + bridge = OperatorBridge() + resolved = bridge.resolve_override( + OperatorOverride.for_target( + semantic_op="selected_logprob", + backend_id="rlkernel.reference_logp", + target="both", + ), + requirements={ + "rollout": _requirements(ScoreSide.ROLLOUT), + "training": _requirements(ScoreSide.TRAINING), + }, + strict=True, + ) + instances = { + target: bridge.instantiate(resolved, target=target) for target in ("rollout", "training") + } + provenance = { + target: bridge.instance_provenance( + resolved, + target=target, + instance=instances[target], + ) + for target in ("rollout", "training") + } + return resolved, instances, provenance + + +def _materialization(case: ExperimentCase): + backend = str(case.requested["logp"]["backend"]) + backends = {"rollout": backend, "training": backend} + return RuntimeTools().materialize( + case, + CpuSmokeMaterializer( + requested_operator_backends=backends, + actual_operator_backends=backends, + ), + ) + + +def _spec(side: ScoreSide) -> ScorerSpec: + identity = _identity() + return ScorerSpec( + side=side, + backend_id="fixed_cpu_teacher_forcing", + dtype="float32", + device="cpu", + world_size=1, + topology=_topology(side), + construction_options={ + "checkpoint_id": identity.checkpoint_id, + "model_version": identity.model_version, + "pre_update_state": identity.pre_update_state, + "teacher_forcing": True, + "use_cache": False, + }, + operator_overrides={"selected_logprob": "rlkernel.reference_logp"}, + ) + + +def _bound_smoke_case(scenario: str) -> tuple[ExperimentCase, OperatorSelection]: + threshold_offset = resolve_logprob_threshold("float32") * 4.0 + if scenario == "reference-reference": + rollout_backend = SMOKE_ONLY_LOGP_REFERENCE_BACKEND_ID + training_backend = SMOKE_ONLY_LOGP_REFERENCE_BACKEND_ID + rollout_options = {} + training_options = {} + elif scenario == "reference-offset": + rollout_backend = SMOKE_ONLY_LOGP_REFERENCE_BACKEND_ID + training_backend = SMOKE_ONLY_LOGP_OFFSET_BACKEND_ID + rollout_options = {} + training_options = {"offset": threshold_offset} + elif scenario == "offset-offset": + rollout_backend = SMOKE_ONLY_LOGP_OFFSET_BACKEND_ID + training_backend = SMOKE_ONLY_LOGP_OFFSET_BACKEND_ID + rollout_options = {"offset": threshold_offset} + training_options = {"offset": threshold_offset} + else: # pragma: no cover - test helper contract + raise ValueError(f"unknown scenario: {scenario}") + selection = OperatorSelection( + rollout_backend=rollout_backend, + training_backend=training_backend, + rollout_options=rollout_options, + training_options=training_options, + ) + case = bind_operator_selection( + _case(case_id=f"case-{scenario}", backend=rollout_backend), + selection, + ) + return case, selection + + +def _write_required_artifacts( + store: ArtifactStore, + attempt_dir: Path, + *, + case_id: str = "case-1", + omit: frozenset[str] = frozenset(), + rollout_logprobs: torch.Tensor | None = None, + training_logprobs: torch.Tensor | None = None, + active_mask: torch.Tensor | None = None, + threshold: float = 0.05, +) -> None: + attempt_id = attempt_dir.name + json_values = { + "requested.json": { + "schema_version": "cross_config.requested.v1", + "case_id": case_id, + "attempt_id": attempt_id, + "case": {"case_id": case_id}, + }, + "materialized.json": { + "schema_version": "cross_config.materialized_envelope.v1", + "case_id": case_id, + "attempt_id": attempt_id, + "materialized_case": {"case": {"case_id": case_id}}, + }, + "actual.json": { + "schema_version": "cross_config.actual.v1", + "case_id": case_id, + "attempt_id": attempt_id, + "rollout": {}, + "training": {}, + }, + "identity.json": { + "schema_version": "cross_config.identity_envelope.v1", + "case_id": case_id, + "attempt_id": attempt_id, + "identity": {"checkpoint_id": "tiny"}, + }, + "comparison.json": { + "schema_version": "cross_config.alignment_result.v1", + "case_id": case_id, + "attempt_id": attempt_id, + "status": "pass", + "comparable": True, + "passed": True, + }, + } + for name, value in json_values.items(): + if name not in omit: + store.write_json(attempt_dir, name, value) + + rollout = rollout_logprobs if rollout_logprobs is not None else torch.tensor([-1.0, -2.0, -3.0]) + training = training_logprobs if training_logprobs is not None else rollout.clone() + active = active_mask if active_mask is not None else torch.tensor([True, True, True]) + mismatch = recompute_mismatch_mask(rollout, training, active, threshold) + tensor_values = { + "score_rollout.pt": { + "selected_logprobs": rollout, + "active_mask": active, + }, + "score_training.pt": { + "selected_logprobs": training, + "active_mask": active, + }, + "token_diffs.pt": { + "rollout_logprobs": rollout, + "training_logprobs": training, + "active_mask": active, + "absolute_diff": torch.abs(training - rollout), + "mismatch_mask": mismatch, + }, + } + for name, tensors in tensor_values.items(): + if name not in omit: + store.write_tensor_bundle( + attempt_dir, + name, + tensors, + metadata={ + "case_id": case_id, + "attempt_id": attempt_id, + "artifact": name, + "fixed_threshold": threshold, + }, + ) + + +def _complete_attempt( + store: ArtifactStore, + *, + experiment_id: str = "experiment-1", + case_id: str = "case-1", + **artifact_options, +) -> Path: + attempt_dir = store.create_attempt(experiment_id, case_id) + _write_required_artifacts(store, attempt_dir, case_id=case_id, **artifact_options) + store.complete_attempt( + attempt_dir, + summary={ + "schema_version": "cross_config.complete.v1", + "case_id": case_id, + "attempt_id": attempt_dir.name, + "status": "pass", + }, + ) + return attempt_dir + + +@pytest.mark.smoke_operator +def test_cpu_smoke_cases_preserve_read_only_scoring_and_exact_provenance(tmp_path: Path): + store = ArtifactStore(tmp_path) + batch = _batch() + inputs_before = batch.input_ids.clone() + expected = { + "reference-reference": (True, 0), + "reference-offset": (False, int(batch.active_mask.sum().item())), + "offset-offset": (True, 0), + } + + for scenario, (expected_pass, expected_mismatches) in expected.items(): + case, selection = _bound_smoke_case(scenario) + result = run_cpu_case( + store, + case, + batch, + selection, + allow_smoke_operators=True, + strict=True, + timeout_seconds=5.0, + resume=False, + ) + + assert result.resumed is False + assert result.alignment is not None + assert result.alignment.passed is expected_pass + assert result.alignment.mismatch_count == expected_mismatches + assert result.rollout_score is not None + assert result.training_score is not None + assert result.rollout_score.selected_logprobs.device.type == "cpu" + assert result.training_score.selected_logprobs.device.type == "cpu" + assert result.rollout_score.scorer.device == "cpu" + assert result.training_score.scorer.device == "cpu" + + guard = result.training_score.provenance.evidence["scoring_guard"] + assert guard == { + "model_state_verified": True, + "model_eval": True, + "no_grad": True, + "optimizer_step": False, + "model_modes_restored": True, + "model_state_unchanged": True, + } + assert result.rollout_score.provenance.evidence["scoring_guard"] == guard + assert result.training_score.provenance.evidence["rank_metadata"][0]["batch_ranges"] == ( + (0, 2), + (2, 3), + ) + rollout_state = result.rollout_score.provenance.evidence["model_state_fingerprint"] + training_state = result.training_score.provenance.evidence["model_state_fingerprint"] + assert rollout_state == training_state + + actual = json.loads((result.attempt_dir / "actual.json").read_text(encoding="utf-8")) + assert actual["operator_source"] == "exact_resolution_and_instance" + for target, backend in ( + ("rollout", selection.rollout_backend), + ("training", selection.training_backend), + ): + operator = actual[target]["actual"]["operators"]["selected_logprob"] + assert operator["backend_id"] == backend + assert operator["descriptor_fingerprint"] + assert operator["implementation_fingerprint"] + assert operator["instance_fingerprint"] + assert (result.attempt_dir / "COMPLETE").is_file() + + assert torch.equal(batch.input_ids, inputs_before) + + +@pytest.mark.smoke_operator +def test_runner_resumes_valid_attempt_and_retries_after_identity_or_tensor_change( + tmp_path: Path, + monkeypatch, +): + store = ArtifactStore(tmp_path) + case, selection = _bound_smoke_case("reference-reference") + batch = _batch() + + def run(): + return run_cpu_case( + store, + case, + batch, + selection, + allow_smoke_operators=True, + strict=True, + timeout_seconds=5.0, + resume=True, + ) + + first = run() + resumed = run() + assert first.attempt_id == "attempt-0001" + assert resumed.resumed is True + assert resumed.attempt_id == first.attempt_id + assert resumed.rollout_score is None + + token_path = first.attempt_dir / "token_diffs.pt" + payload = torch.load(token_path, map_location="cpu", weights_only=True) + payload["tensors"]["mismatch_mask"] = torch.ones_like(payload["tensors"]["mismatch_mask"]) + torch.save(payload, token_path) + + retried = run() + assert retried.resumed is False + assert retried.attempt_id == "attempt-0002" + assert (retried.attempt_dir / "COMPLETE").is_file() + + original_apply_fp32 = SmokeOnlyLogpReference.apply_fp32 + + def equivalent_apply_fp32(self, logits, token_ids, active_mask=None): + return original_apply_fp32(self, logits, token_ids, active_mask=active_mask) + + monkeypatch.setattr(SmokeOnlyLogpReference, "apply_fp32", equivalent_apply_fp32) + implementation_changed = run() + assert implementation_changed.resumed is False + assert implementation_changed.attempt_id == "attempt-0003" + before = json.loads((retried.attempt_dir / "actual.json").read_text(encoding="utf-8")) + after = json.loads( + (implementation_changed.attempt_dir / "actual.json").read_text(encoding="utf-8") + ) + assert ( + before["rollout"]["actual"]["operators"]["selected_logprob"]["implementation_fingerprint"] + != after["rollout"]["actual"]["operators"]["selected_logprob"]["implementation_fingerprint"] + ) + attempts = sorted(path.name for path in retried.attempt_dir.parent.iterdir()) + assert attempts == ["attempt-0001", "attempt-0002", "attempt-0003"] + + +@pytest.mark.parametrize( + ("mode", "error_type", "message"), + [ + ("failure", ChildScoringError, "intentional scorer failure"), + ("timeout", ScoringTimeoutError, "stopped children"), + ("missing-rank", RankCompletenessError, r"missing=\[0\]"), + ("duplicate-rank", RankCompletenessError, "duplicate ranks"), + ], +) +def test_runner_supervision_fails_closed_and_cleans_children( + tmp_path: Path, + mode: str, + error_type: type[Exception], + message: str, +): + case = _case(case_id=f"case-{mode}") + resolved, instances, provenance = _operators() + if mode == "failure": + rollout = FailingScorer(_spec(ScoreSide.ROLLOUT), ()) + training = FixedRankScorer(_spec(ScoreSide.TRAINING), (0,)) + timeout = 5.0 + elif mode == "timeout": + rollout = SlowScorer(_spec(ScoreSide.ROLLOUT), (0,)) + training = SlowScorer(_spec(ScoreSide.TRAINING), (0,)) + timeout = 0.1 + elif mode == "missing-rank": + rollout = FixedRankScorer(_spec(ScoreSide.ROLLOUT), ()) + training = FixedRankScorer(_spec(ScoreSide.TRAINING), (0,)) + timeout = 5.0 + else: + rollout = FixedRankScorer(_spec(ScoreSide.ROLLOUT), (0, 0)) + training = FixedRankScorer(_spec(ScoreSide.TRAINING), (0,)) + timeout = 5.0 + + runner = PairedRunner(ArtifactStore(tmp_path), timeout_seconds=timeout) + with pytest.raises(error_type, match=message): + runner.run( + case, + _materialization(case), + _batch(), + rollout, + training, + resolved, + instances, + provenance, + timeout_seconds=timeout, + ) + + assert runner.active_child_pids == () + attempt_dir = tmp_path / case.experiment_id / "cases" / case.case_id / "attempt-0001" + assert attempt_dir.is_dir() + assert not (attempt_dir / "COMPLETE").exists() + assert not list(attempt_dir.glob(".paired-runner-*")) + + +def test_artifacts_are_append_only_and_complete_marker_is_published_last(tmp_path: Path): + store = ArtifactStore(tmp_path) + attempt_dir = store.create_attempt("experiment-1", "case-1") + _write_required_artifacts( + store, + attempt_dir, + omit=frozenset({"token_diffs.pt"}), + ) + requested_path = attempt_dir / "requested.json" + rollout_path = attempt_dir / "score_rollout.pt" + requested_before = requested_path.read_bytes() + rollout_before = rollout_path.read_bytes() + + with pytest.raises(ArtifactError, match="refusing to overwrite"): + store.write_json(attempt_dir, "requested", {"case_id": "changed"}) + with pytest.raises(ArtifactError, match="refusing to overwrite"): + store.write_tensor_bundle( + attempt_dir, + "score_rollout", + {"selected_logprobs": torch.tensor([0.0])}, + ) + assert requested_path.read_bytes() == requested_before + assert rollout_path.read_bytes() == rollout_before + + summary = { + "schema_version": "cross_config.complete.v1", + "case_id": "case-1", + "attempt_id": attempt_dir.name, + "status": "pass", + } + with pytest.raises(ArtifactError, match=r"missing artifacts.*token_diffs\.pt"): + store.complete_attempt(attempt_dir, summary=summary) + assert not (attempt_dir / "COMPLETE").exists() + + _write_required_artifacts( + store, + attempt_dir, + omit=frozenset(_JSON_ARTIFACTS + _TENSOR_ARTIFACTS[:-1]), + ) + marker = store.complete_attempt(attempt_dir, summary=summary) + store.validate_completed_attempt(attempt_dir, expected_case_id="case-1") + payload_times = [ + (attempt_dir / name).stat().st_mtime_ns for name in _JSON_ARTIFACTS + _TENSOR_ARTIFACTS + ] + assert marker.stat().st_mtime_ns >= max(payload_times) + assert json.loads(marker.read_text(encoding="utf-8")) == summary + assert not list(attempt_dir.glob(".COMPLETE.*")) + with pytest.raises(ArtifactError, match="refusing to overwrite"): + store.complete_attempt(attempt_dir, summary=summary) + + next_attempt = store.create_attempt("experiment-1", "case-1") + assert next_attempt.name == "attempt-0002" + + +def test_resume_uses_newest_valid_attempt_and_tensors_support_offline_recompute(tmp_path: Path): + store = ArtifactStore(tmp_path) + rollout = torch.tensor([-1.0, -2.0, -3.0]) + training = torch.tensor([-1.01, -2.20, -2.50]) + active = torch.tensor([True, True, False]) + older = _complete_attempt( + store, + rollout_logprobs=rollout, + training_logprobs=training, + active_mask=active, + threshold=0.05, + ) + newer = _complete_attempt(store) + partial = store.create_attempt("experiment-1", "case-1") + store.write_json(partial, "requested", {"case_id": "case-1"}) + + assert store.completed_attempt("experiment-1", "case-1") == newer + (newer / "COMPLETE").write_text("{not-json", encoding="utf-8") + assert store.completed_attempt("experiment-1", "case-1") == older + + token_payload = store.load_tensor_bundle(older / "token_diffs.pt") + tensors = token_payload["tensors"] + recomputed = recompute_mismatch_mask( + tensors["rollout_logprobs"], + tensors["training_logprobs"], + tensors["active_mask"], + token_payload["metadata"]["fixed_threshold"], + ) + assert torch.equal(recomputed, tensors["mismatch_mask"]) + assert torch.equal(recomputed, torch.tensor([False, True, False])) + assert all(tensor.device.type == "cpu" for tensor in tensors.values()) + assert partial.name == "attempt-0003" diff --git a/tests/test_cross_config_runtime.py b/tests/test_cross_config_runtime.py new file mode 100644 index 00000000..1523c66b --- /dev/null +++ b/tests/test_cross_config_runtime.py @@ -0,0 +1,759 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +from __future__ import annotations + +import inspect +from dataclasses import replace +from pathlib import Path + +import pytest +import torch + +from rl_engine.alignment.cross_config.operators import ( + OperatorBridge, + OperatorOverride, + selected_logprobs_with_operator, +) +from rl_engine.alignment.cross_config.planner import V1_KNOBS +from rl_engine.alignment.cross_config.runtime import ( + AdapterMaterialization, + KnobApplication, + RuntimeBinding, + RuntimeMaterializationError, + RuntimeTools, +) +from rl_engine.alignment.cross_config.schema import ( + ExperimentCase, + IsolationScope, + MaterializationStatus, + SemanticIdentitySpec, +) +from rl_engine.alignment.testing.cpu_cross_config import CpuSmokeMaterializer +from rl_engine.alignment.testing.smoke_ops import ( + SMOKE_ONLY_LOGP_OFFSET_BACKEND_ID, + SMOKE_ONLY_LOGP_REFERENCE_BACKEND_ID, + SmokeOnlyLogpOffset, + register_smoke_operators, +) +from rl_engine.kernels.ops.pytorch.loss.logp import NativeLogpOp +from rl_engine.kernels.semantic_registry import ( + OperatorRequirements, + OperatorResolutionError, + OperatorResolutionPolicy, + SemanticOperatorCatalog, +) +from rl_engine.kernels.semantic_registry import ( + implementation_fingerprint as fingerprint_implementation, +) +from rl_engine.testing import selected_logprobs_reference + +_REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +_TEMPORARY_DOCSTRING = "TEMPORARY TEST SCAFFOLD - NOT A PRODUCTION RL-KERNEL OPERATOR" +_TOPOLOGY = { + "world_size": 1, + "tensor_parallel_size": 1, + "context_parallel_size": 1, + "sharding": "unsharded", +} + + +def _identity() -> SemanticIdentitySpec: + return SemanticIdentitySpec( + checkpoint_id="tiny-cpu-checkpoint", + model_version="weights-v1", + tokenizer_policy="synthetic-tokenizer-v1", + token_ids=((1, 2, 3),), + selected_token_ids=((0, 2, 3),), + active_mask=((False, True, True),), + attention_mask=((True, True, True),), + pre_update_state="iteration-0", + ) + + +def _requested(**overrides): + requested = { + "batch": {"size": 2}, + "rollout": { + "tensor_parallel_size": 1, + "context_parallel_size": 1, + "dtype": "float32", + "enable_prefix_caching": False, + "enforce_eager": True, + }, + "training": { + "attention_backend": "eager", + "compute_dtype": "float32", + "sharding": "unsharded", + }, + "logp": {"backend": "rlkernel.reference_logp"}, + } + for path, value in overrides.items(): + current = requested + parts = path.split(".") + for part in parts[:-1]: + current = current[part] + current[parts[-1]] = value + return requested + + +def _case( + *, + case_id: str = "case-1", + changed_paths=(), + requested=None, + execution_binding=None, +) -> ExperimentCase: + return ExperimentCase( + case_id=case_id, + experiment_id="runtime-test", + scenario_id="S0", + identity=_identity(), + requested=requested or _requested(), + changed_paths=changed_paths, + execution_binding=execution_binding or {}, + contract_fingerprint="contract-sha", + scenario_fingerprint="scenario-sha", + ) + + +def _value_at(requested, path: str): + current = requested + for part in path.split("."): + current = current[part] + return current + + +def _readback(requested): + return {path: _value_at(requested, path) for path in V1_KNOBS if path != "batch.size"} + + +class _RuntimeTestAdapter: + """Small observable fake kept beside the lifecycle tests that need it.""" + + runtime_kind = "test_runtime" + + def __init__(self, *, actual_readback=None): + self.actual_readback = dict(actual_readback or {}) + + @property + def implementation_fingerprint(self): + return fingerprint_implementation( + type(self), + instance=self, + entrypoints=("materialize",), + ) + + def materialize(self, normalized, descriptors): + applications = [] + for path, descriptor in descriptors.items(): + requested = _value_at(normalized, path) + materialized = requested + actual = self.actual_readback.get(path) + status = MaterializationStatus.UNOBSERVABLE + reason = "no runtime readback is available" + + unsupported = (path == "rollout.context_parallel_size" and requested != 1) or ( + path == "training.sharding" and requested != "unsharded" + ) + if unsupported: + materialized = actual = None + status = MaterializationStatus.UNSUPPORTED + reason = "the test adapter does not support this topology" + elif path == "batch.size": + actual = requested + status = MaterializationStatus.APPLIED + reason = "batch size is observed at scorer invocation" + elif path in self.actual_readback: + status = ( + MaterializationStatus.APPLIED + if actual == requested + else MaterializationStatus.FALLBACK + ) + reason = "runtime readback was captured" + + applications.append( + KnobApplication( + path=path, + requested=requested, + materialized=materialized, + actual=actual, + lifecycle=descriptor.lifecycle, + status=status, + evidence={"reason": reason}, + critical=descriptor.critical, + ) + ) + + backend = _value_at(normalized, "logp.backend") + return AdapterMaterialization( + applications=tuple(applications), + binding=RuntimeBinding( + batch_size=_value_at(normalized, "batch.size"), + side_configs={"rollout": {}, "training": {}}, + topology={ + "rollout": { + "world_size": 1, + "tensor_parallel_size": _value_at( + normalized, "rollout.tensor_parallel_size" + ), + "context_parallel_size": _value_at( + normalized, "rollout.context_parallel_size" + ), + }, + "training": { + "world_size": 1, + "sharding": _value_at(normalized, "training.sharding"), + }, + }, + scorer={}, + operator_backends={"rollout": backend, "training": backend}, + runtime_kind=self.runtime_kind, + ), + ) + + +def _cpu_materializer() -> CpuSmokeMaterializer: + backends = { + "rollout": "rlkernel.reference_logp", + "training": "rlkernel.reference_logp", + } + return CpuSmokeMaterializer( + requested_operator_backends=backends, + actual_operator_backends=backends, + ) + + +def _requirements( + *, + device: str = "cpu", +) -> OperatorRequirements: + return OperatorRequirements( + device=device, + dtype="float32", + topology=_TOPOLOGY, + alignment_properties={"deterministic": True}, + ) + + +def _catalog() -> SemanticOperatorCatalog: + """Clone repository descriptors so each test owns registration state.""" + + return SemanticOperatorCatalog(OperatorBridge().catalog.backend_descriptors()) + + +def test_cpu_materialization_records_all_ten_knobs_across_three_stages(): + case = _case() + materialization = RuntimeTools().materialize(case, _cpu_materializer()) + applications = {application.path: application for application in materialization.applications} + + assert len(V1_KNOBS) == 10 + assert set(applications) == set(V1_KNOBS) + assert materialization.materialized_case.status is MaterializationStatus.APPLIED + assert materialization.executable_in_strict_mode + RuntimeTools.require_executable(materialization, strict=True) + + for path, descriptor in V1_KNOBS.items(): + application = applications[path] + assert application.requested == _value_at(case.requested, path) + assert application.lifecycle is descriptor.lifecycle + assert application.status is MaterializationStatus.APPLIED + assert application.evidence["reason"] + + provenance = materialization.provenance + assert provenance.requested == case.requested + assert provenance.normalized == case.requested + assert provenance.materialized["batch"]["size"] == 2 + assert provenance.materialized["rollout"] == { + "tensor_parallel_size": 1, + "context_parallel_size": 1, + "dtype": "float32", + "enable_prefix_caching": False, + "enforce_eager": True, + } + assert provenance.materialized["training"] == { + "attention_backend": "eager", + "compute_dtype": "float32", + "sharding": "unsharded", + } + assert provenance.materialized["logp"]["backend"] == { + "rollout": "rlkernel.reference_logp", + "training": "rlkernel.reference_logp", + } + assert provenance.actual == provenance.materialized + assert provenance.implementation_fingerprint == _cpu_materializer().implementation_fingerprint + assert provenance.evidence["adapter_implementation_fingerprint"] == ( + provenance.implementation_fingerprint + ) + + binding = materialization.binding + assert binding.runtime_kind == "cpu_smoke" + assert binding.side_configs["rollout"]["device"] == "cpu" + assert binding.side_configs["training"]["device"] == "cpu" + assert binding.side_configs["training"]["dtype"] == "float32" + assert binding.side_configs["rollout"] == { + "device": "cpu", + "dtype": "float32", + "enable_prefix_caching": False, + "enforce_eager": True, + } + assert binding.topology["rollout"] == { + "world_size": 1, + "tensor_parallel_size": 1, + "context_parallel_size": 1, + } + assert binding.topology["training"] == {"world_size": 1, "sharding": "unsharded"} + assert binding.scorer == { + "mode": "reference", + "use_cache": False, + "attention_backend": "eager", + "output_dtype": "float32", + } + with pytest.raises(TypeError): + binding.side_configs["rollout"]["device"] = "cuda" + with pytest.raises(TypeError): + applications["batch.size"].evidence["reason"] = "changed after fingerprinting" + + +def test_lifecycle_fingerprints_allow_request_reuse_and_isolate_engine_and_process_changes( + monkeypatch, +): + tools = RuntimeTools() + baseline_case = _case(case_id="baseline") + baseline_adapter = _RuntimeTestAdapter(actual_readback=_readback(baseline_case.requested)) + baseline = tools.materialize( + baseline_case, + baseline_adapter, + ) + + batch_requested = _requested(**{"batch.size": 1}) + batch = tools.materialize( + _case( + case_id="batch", + requested=batch_requested, + changed_paths=("batch.size",), + ), + _RuntimeTestAdapter(actual_readback=_readback(batch_requested)), + ) + assert batch.materialized_case.isolation_scope is IsolationScope.REQUEST + assert batch.materialized_case.construction_fingerprint == ( + baseline.materialized_case.construction_fingerprint + ) + assert batch.materialized_case.distributed_context_fingerprint == ( + baseline.materialized_case.distributed_context_fingerprint + ) + assert batch.materialized_case.process_fingerprint == ( + baseline.materialized_case.process_fingerprint + ) + assert tools.can_reuse(baseline, batch) + + dtype_requested = _requested(**{"rollout.dtype": "bfloat16"}) + dtype = tools.materialize( + _case( + case_id="dtype", + requested=dtype_requested, + changed_paths=("rollout.dtype",), + ), + _RuntimeTestAdapter(actual_readback=_readback(dtype_requested)), + ) + assert dtype.materialized_case.isolation_scope is IsolationScope.ENGINE_CONSTRUCTION + assert dtype.materialized_case.construction_fingerprint != ( + baseline.materialized_case.construction_fingerprint + ) + assert dtype.materialized_case.process_fingerprint == ( + baseline.materialized_case.process_fingerprint + ) + assert not tools.can_reuse(baseline, dtype) + + topology_requested = _requested(**{"rollout.tensor_parallel_size": 2}) + topology = tools.materialize( + _case( + case_id="topology", + requested=topology_requested, + changed_paths=("rollout.tensor_parallel_size",), + ), + _RuntimeTestAdapter(actual_readback=_readback(topology_requested)), + ) + assert topology.materialized_case.isolation_scope is IsolationScope.PROCESS + assert topology.materialized_case.process_fingerprint != ( + baseline.materialized_case.process_fingerprint + ) + assert not tools.can_reuse(baseline, topology) + + rebound_case = replace( + baseline_case, + case_id="rebound", + execution_binding={"operator_case": {"rollout_options": {"offset": 0.1}}}, + ) + rebound = tools.materialize( + rebound_case, + _RuntimeTestAdapter(actual_readback=_readback(rebound_case.requested)), + ) + assert rebound.materialized_case.construction_fingerprint == ( + baseline.materialized_case.construction_fingerprint + ) + assert not tools.can_reuse(baseline, rebound) + + changed_identity_case = replace( + baseline_case, + case_id="changed-identity", + identity=replace(baseline_case.identity, pre_update_state="iteration-1"), + ) + changed_identity = tools.materialize(changed_identity_case, baseline_adapter) + assert changed_identity.materialized_case.construction_fingerprint == ( + baseline.materialized_case.construction_fingerprint + ) + assert not tools.can_reuse(baseline, changed_identity) + + original_materialize = _RuntimeTestAdapter.materialize + + def materialize_with_same_result(self, normalized, descriptors): + return original_materialize(self, normalized, descriptors) + + monkeypatch.setattr( + _RuntimeTestAdapter, + "materialize", + materialize_with_same_result, + ) + changed_adapter = tools.materialize( + baseline_case, + _RuntimeTestAdapter(actual_readback=_readback(baseline_case.requested)), + ) + assert changed_adapter.provenance.actual == baseline.provenance.actual + assert ( + changed_adapter.provenance.implementation_fingerprint + != baseline.provenance.implementation_fingerprint + ) + assert not tools.can_reuse(baseline, changed_adapter) + + +def test_materialization_fails_closed_for_fallback_unobservable_and_unsupported_paths(): + tools = RuntimeTools() + + fallback_requested = _requested(**{"training.attention_backend": "flash_attention_2"}) + fallback_readback = _readback(fallback_requested) + fallback_readback["training.attention_backend"] = "eager" + fallback = tools.materialize( + _case( + case_id="fallback", + requested=fallback_requested, + changed_paths=("training.attention_backend",), + ), + _RuntimeTestAdapter(actual_readback=fallback_readback), + ) + assert fallback.materialized_case.status is MaterializationStatus.FALLBACK + with pytest.raises(RuntimeMaterializationError, match="training.attention_backend"): + tools.require_executable(fallback, strict=True) + tools.require_executable(fallback, strict=False) + + unobservable = tools.materialize( + _case(case_id="unobservable"), + _RuntimeTestAdapter(), + ) + assert unobservable.materialized_case.status is MaterializationStatus.UNOBSERVABLE + with pytest.raises(RuntimeMaterializationError, match="no runtime readback"): + tools.require_executable(unobservable, strict=False) + + unsupported_requested = _requested( + **{ + "rollout.context_parallel_size": 4, + "training.sharding": "fsdp", + } + ) + unsupported = tools.materialize( + _case( + case_id="unsupported", + requested=unsupported_requested, + changed_paths=("rollout.context_parallel_size", "training.sharding"), + ), + _RuntimeTestAdapter(), + ) + unsupported_paths = { + application.path + for application in unsupported.applications + if application.status is MaterializationStatus.UNSUPPORTED + } + assert unsupported_paths == {"rollout.context_parallel_size", "training.sharding"} + assert unsupported.materialized_case.status is MaterializationStatus.UNSUPPORTED + with pytest.raises(RuntimeMaterializationError, match="rollout.context_parallel_size"): + tools.require_executable(unsupported, strict=False) + + cpu_only_requested = _requested(**{"rollout.tensor_parallel_size": 2}) + cpu_only = tools.materialize( + _case( + case_id="cpu-only", + requested=cpu_only_requested, + changed_paths=("rollout.tensor_parallel_size",), + ), + _cpu_materializer(), + ) + assert cpu_only.materialized_case.status is MaterializationStatus.UNSUPPORTED + assert cpu_only.binding.side_configs["rollout"]["device"] == "cpu" + assert cpu_only.binding.side_configs["training"]["device"] == "cpu" + + +def test_operator_binding_selects_rollout_training_and_both_without_side_leakage(): + bridge = OperatorBridge() + requirements = { + "rollout": _requirements(), + "training": _requirements(), + } + assert requirements["rollout"].to_dict()["schema_version"] == ( + "rlkernel.semantic_operator.requirements.v1" + ) + + for target, expected_targets in ( + ("rollout", {"rollout"}), + ("training", {"training"}), + ("both", {"rollout", "training"}), + ): + resolved = bridge.resolve_override( + OperatorOverride.for_target( + semantic_op="selected_logprob", + backend_id="rlkernel.reference_logp", + target=target, + ), + requirements=requirements, + strict=True, + ) + selected_targets = { + side for side in ("rollout", "training") if resolved.for_target(side) is not None + } + assert selected_targets == expected_targets + + instances = {} + for side in expected_targets: + resolution = resolved.for_target(side) + assert resolution is not None + assert resolution.to_dict()["schema_version"] == ( + "rlkernel.semantic_operator.resolution.v1" + ) + assert resolution.descriptor.to_dict()["schema_version"] == ( + "rlkernel.semantic_operator.backend_descriptor.v1" + ) + assert resolution.trace.to_dict()["schema_version"] == ( + "rlkernel.semantic_operator.resolution_trace.v1" + ) + instance = bridge.instantiate(resolved, target=side) + instances[side] = instance + assert isinstance(instance, NativeLogpOp) + provenance = bridge.instance_provenance( + resolved, + target=side, + instance=instance, + ) + assert provenance.backend_id == "rlkernel.reference_logp" + assert provenance.target == side + assert provenance.instance_fingerprint + assert provenance.to_dict()["schema_version"] == ( + "rlkernel.semantic_operator.instance_provenance.v1" + ) + with pytest.raises(TypeError): + provenance.factory_options["unexpected"] = True + if target == "both": + assert instances["rollout"] is not instances["training"] + + catalog = _catalog() + descriptor = catalog.backend_descriptor("selected_logprob", "rlkernel.reference_logp") + assert descriptor is not None + catalog.register_backend( + replace(descriptor, supported_topologies={"*": "*"}), + replace=True, + ) + asymmetric = OperatorBridge(catalog).resolve_override( + OperatorOverride.for_target( + semantic_op="selected_logprob", + backend_id="rlkernel.reference_logp", + target="both", + ), + requirements={ + "rollout": OperatorRequirements( + device="cpu", + dtype="float32", + topology={"world_size": 2, "tensor_parallel_size": 2}, + ), + "training": OperatorRequirements( + device="cpu", + dtype="float32", + topology={"world_size": 1, "sharding": "fsdp"}, + ), + }, + ) + assert asymmetric.rollout is not None and asymmetric.training is not None + assert asymmetric.rollout.requirements.topology != asymmetric.training.requirements.topology + + session = catalog.session() + with pytest.raises(OperatorResolutionError, match="not registered") as unsupported: + session.resolve( + semantic_op="selected_logprob", + requested_backend="missing.backend", + target="rollout", + requirements=_requirements(), + strict=True, + ) + assert unsupported.value.trace.status == "unsupported" + assert unsupported.value.trace.fallback_attempts == () + + native_requirements = OperatorRequirements( + device="cpu", + dtype="float32", + topology=_TOPOLOGY, + ) + with pytest.raises(OperatorResolutionError, match="not exactly observable"): + session.resolve( + semantic_op="selected_logprob", + requested_backend="native", + target="training", + requirements=native_requirements, + strict=True, + ) + native = session.resolve( + semantic_op="selected_logprob", + requested_backend="native", + target="training", + requirements=native_requirements, + strict=False, + ) + assert native.trace.status == "unobservable" + assert native.trace.concrete_backend is None + + +@pytest.mark.smoke_operator +def test_smoke_package_is_temporary_cpu_only_and_disabled_without_two_explicit_opt_ins(): + from rl_engine.alignment.testing.smoke_ops import ( + smoke_only_logp_offset, + smoke_only_logp_reference, + ) + + assert inspect.getdoc(smoke_only_logp_reference) == _TEMPORARY_DOCSTRING + assert inspect.getdoc(smoke_only_logp_offset) == _TEMPORARY_DOCSTRING + + manifest = ( + _REPOSITORY_ROOT / "rl_engine/alignment/testing/smoke_ops/SMOKE_OPERATORS.md" + ).read_text(encoding="utf-8") + for required_text in ( + "temporary test scaffolding", + "smoke_only_logp_reference.py", + "smoke_only_logp_offset.py", + "allow_smoke_operators=True", + "delete this package", + ): + assert required_text in manifest + + catalog = _catalog() + for backend_id in ( + SMOKE_ONLY_LOGP_REFERENCE_BACKEND_ID, + SMOKE_ONLY_LOGP_OFFSET_BACKEND_ID, + ): + assert catalog.backend_descriptor("selected_logprob", backend_id) is None + + with pytest.raises(PermissionError, match="allow_smoke_operators=True"): + register_smoke_operators(catalog) + + descriptors = register_smoke_operators(catalog, allow_smoke_operators=True) + assert {descriptor.backend_id for descriptor in descriptors} == { + SMOKE_ONLY_LOGP_REFERENCE_BACKEND_ID, + SMOKE_ONLY_LOGP_OFFSET_BACKEND_ID, + } + for descriptor in descriptors: + assert descriptor.supported_devices == frozenset({"cpu"}) + assert descriptor.is_smoke_only is True + disabled_session = catalog.session( + OperatorResolutionPolicy(strict=True, allow_test_backends=False) + ) + with pytest.raises(OperatorResolutionError, match="explicit opt-in"): + disabled_session.resolve( + semantic_op="selected_logprob", + requested_backend=descriptor.backend_id, + target="training", + requirements=_requirements(), + ) + enabled_session = catalog.session( + OperatorResolutionPolicy(strict=True, allow_test_backends=True) + ) + with pytest.raises(OperatorResolutionError) as error: + enabled_session.resolve( + semantic_op="selected_logprob", + requested_backend=descriptor.backend_id, + target="training", + requirements=_requirements(device="cuda"), + ) + failed = { + decision.capability + for decision in error.value.trace.capability_decisions + if not decision.passed + } + assert failed == {"device"} + + assert SmokeOnlyLogpOffset().offset == 0.0 + with pytest.raises(PermissionError, match="allow_smoke_operators=True"): + SmokeOnlyLogpOffset(offset=0.01) + + +@pytest.mark.smoke_operator +def test_explicit_smoke_opt_in_runs_both_sides_on_cpu_with_sealed_provenance(): + catalog = _catalog() + register_smoke_operators(catalog, allow_smoke_operators=True) + bridge = OperatorBridge( + catalog, + policy=OperatorResolutionPolicy(strict=True, allow_test_backends=True), + ) + resolved = bridge.resolve_override( + OperatorOverride.for_target( + semantic_op="selected_logprob", + backend_id=SMOKE_ONLY_LOGP_REFERENCE_BACKEND_ID, + target="both", + ), + requirements={ + "rollout": _requirements(), + "training": _requirements(), + }, + strict=True, + ) + instances = { + target: bridge.instantiate(resolved, target=target) for target in ("rollout", "training") + } + logits = torch.tensor( + [[[1.0, 2.0, -1.0], [0.0, 3.0, 1.0], [2.0, 0.0, 4.0]]], + device="cpu", + ) + token_ids = torch.tensor([[1, -100, 2]], device="cpu") + active_mask = torch.tensor([[True, False, True]], device="cpu") + expected = selected_logprobs_reference(logits, token_ids, mask=active_mask) + + outputs = {} + for target, instance in instances.items(): + output = selected_logprobs_with_operator( + instance, + logits, + token_ids, + active_mask=active_mask, + ) + outputs[target] = output + assert output.device.type == "cpu" + torch.testing.assert_close(output, expected, atol=0.0, rtol=0.0) + assert torch.count_nonzero(output[~active_mask]).item() == 0 + + provenance = bridge.instance_provenance( + resolved, + target=target, + instance=instance, + ) + assert provenance.backend_id == SMOKE_ONLY_LOGP_REFERENCE_BACKEND_ID + assert provenance.target == target + assert provenance.concrete_implementation.endswith("SmokeOnlyLogpReference") + assert provenance.descriptor_fingerprint + assert provenance.instance_fingerprint + + for invalid_temperature in (float("nan"), float("inf"), float("-inf")): + with pytest.raises(ValueError, match="finite and greater than zero"): + selected_logprobs_with_operator( + instances["rollout"], + logits, + token_ids, + active_mask=active_mask, + temperature=invalid_temperature, + ) + + assert instances["rollout"] is not instances["training"] + torch.testing.assert_close(outputs["rollout"], outputs["training"], atol=0.0, rtol=0.0) diff --git a/tests/test_stateless_executor.py b/tests/test_stateless_executor.py index fc6d9b3a..851402f7 100644 --- a/tests/test_stateless_executor.py +++ b/tests/test_stateless_executor.py @@ -172,6 +172,90 @@ def test_executor_runs_full_sequence_forward_with_use_cache_false_and_detaches_o assert not hasattr(model.generation_config, "attn_implementation") +def test_executor_exact_selected_logprob_callable_is_optional_and_injected(): + inputs = _inputs() + logits = _logits_for(inputs) + calls = [] + + def selected_logprob_fn( + shifted_logits, + shifted_labels, + *, + mask, + temperature, + output_dtype, + ): + calls.append((shifted_logits, shifted_labels, mask, temperature, output_dtype)) + reference = selected_logprobs_reference( + shifted_logits, + shifted_labels, + mask=mask, + temperature=temperature, + output_dtype=output_dtype, + ) + return reference + mask.to(dtype=output_dtype) * 0.25 + + default = StatelessForwardExecutor( + FakeReferenceModel(logits), + StatelessForwardConfig(mode="reference"), + ).score(inputs) + injected = StatelessForwardExecutor( + FakeReferenceModel(logits), + StatelessForwardConfig(mode="reference"), + selected_logprob_fn=selected_logprob_fn, + ).score(inputs) + + assert default.reference_logps is not None + assert injected.reference_logps is not None + assert len(calls) == 1 + assert torch.equal(calls[0][2], inputs.completion_mask[:, 1:]) + torch.testing.assert_close( + injected.reference_logps[inputs.completion_mask], + default.reference_logps[inputs.completion_mask] + 0.25, + ) + assert torch.equal( + injected.reference_logps[~inputs.completion_mask], + torch.zeros_like(injected.reference_logps[~inputs.completion_mask]), + ) + + +def test_executor_uses_eval_no_grad_and_restores_mixed_module_modes_read_only(): + inputs = _inputs() + + class ReadOnlyModel(torch.nn.Module): + def __init__(self): + super().__init__() + self.weight = torch.nn.Parameter(torch.tensor(2.0)) + self.register_buffer("counter", torch.tensor(3.0)) + self.child = torch.nn.Linear(1, 1) + + def forward(self, input_ids, attention_mask=None, use_cache=None): + del attention_mask + assert self.training is False + assert self.child.training is False + assert torch.is_grad_enabled() is False + assert use_cache is False + batch, sequence = input_ids.shape + logits = torch.zeros(batch, sequence, 8) + return {"logits": logits + self.weight * 0.0 + self.counter * 0.0} + + model = ReadOnlyModel() + model.train() + model.child.eval() + state_before = {name: value.detach().clone() for name, value in model.state_dict().items()} + + result = StatelessForwardExecutor( + model, + StatelessForwardConfig(mode="reference", attention_backend="eager"), + ).score(inputs) + + assert result.reference_logps is not None + assert result.metrics["model_eval_during_forward"] is True + assert model.training is True + assert model.child.training is False + assert all(torch.equal(state_before[name], value) for name, value in model.state_dict().items()) + + def test_executor_falls_back_for_models_without_use_cache_argument(): inputs = _inputs() executor = StatelessForwardExecutor( diff --git a/tests/test_tolerance_contract.py b/tests/test_tolerance_contract.py index fb429d81..b7b7d8b2 100644 --- a/tests/test_tolerance_contract.py +++ b/tests/test_tolerance_contract.py @@ -3,7 +3,18 @@ from __future__ import annotations -from rl_engine.kernels.gtest.tolerance import load_contract +import hashlib +import inspect +import json + +import pytest +import torch + +from rl_engine.kernels.gtest.tolerance import ( + load_contract, + resolve_logprob_threshold, + tolerance_contract_fingerprint, +) def test_load_contract_contains_expected_operator_classes(): @@ -27,3 +38,41 @@ def test_logprob_bfloat16_tolerance_covers_observed_reference_drift(): tolerance = contract["accuracy"]["default"]["logprob"]["bfloat16"] assert tolerance["atol"] >= 5.0e-2 assert tolerance["rtol"] == 0.0 + + +@pytest.mark.parametrize( + ("dtype", "dtype_name"), + ( + (torch.float32, "float32"), + ("fp32", "float32"), + (torch.bfloat16, "bfloat16"), + ("bf16", "bfloat16"), + (torch.float16, "float16"), + ("fp16", "float16"), + ), +) +def test_resolve_logprob_threshold_reads_current_ws1_absolute_tolerance(dtype, dtype_name): + expected = load_contract()["accuracy"]["default"]["logprob"][dtype_name]["atol"] + + assert resolve_logprob_threshold(dtype) == expected + + +def test_resolve_logprob_threshold_has_no_contract_or_value_override_parameter(): + assert tuple(inspect.signature(resolve_logprob_threshold).parameters) == ("dtype",) + + +def test_resolve_logprob_threshold_rejects_dtype_outside_ws1_contract(): + with pytest.raises(ValueError, match="unsupported WS1 logprob dtype"): + resolve_logprob_threshold(torch.float64) + + +def test_tolerance_contract_fingerprint_is_canonical_content_sha256(): + canonical = json.dumps( + load_contract(), + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + + assert tolerance_contract_fingerprint() == hashlib.sha256(canonical).hexdigest() + assert len(tolerance_contract_fingerprint()) == 64 From 35073a1b0baa13161279492c10b6ceedbce5e9ff Mon Sep 17 00:00:00 2001 From: CyberSecurityErial <2710555967@qq.com> Date: Sun, 19 Jul 2026 08:35:49 +0800 Subject: [PATCH 8/8] docs(alignment): document cross-configuration workflow --- docs/assets/ws2-cross-config-before-after.png | Bin 93735 -> 0 bytes .../cross_config_implementation_report.md | 185 ++++ .../cross_config_logprob_drift_contract.md | 306 ++++++ ...ws2_cross_config_logprob_drift_contract.md | 874 ------------------ 4 files changed, 491 insertions(+), 874 deletions(-) delete mode 100644 docs/assets/ws2-cross-config-before-after.png create mode 100644 docs/design/cross_config_implementation_report.md create mode 100644 docs/design/cross_config_logprob_drift_contract.md delete mode 100644 docs/design/ws2_cross_config_logprob_drift_contract.md diff --git a/docs/assets/ws2-cross-config-before-after.png b/docs/assets/ws2-cross-config-before-after.png deleted file mode 100644 index c83db1cea0624b150234ca0899f789e3bba3cda8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 93735 zcmeEv2{@Ho+drZVl_;}hNHS!br^=W)DzmU<*oLsT*)~%&nF&deqRgbsV+tW5^Gq_d z%`%Vwwb{E)=RNQHo$veVJH6-so~vu0y`N`2>sjlb*1higcQ3aus>&ZArX|L~!8xF) za83;eXDm%x1o6C-QbZUf%kE5-IQ#{+udVU-nSmEvWU7ToR8(sZ{s6C39{%Lom`?_Yp@jSW)X-pJNM1z`%a20fZ0ce~)@K{r4T+|(5_YF>VU z-7Dty*cV}Dg1(?`>4u#i=0S5OOH-Hw=HcBZ9T5m?M@!r9FPb3WaF_{(c$glI?ClXQ z-#0TuSYxJ#X=7_-j(y{9@8}Ea|3GH+Rc%XCM~mH&i0}!2YpnpYG`GOMnO8`3_l}Jb zwlQWV4i-kH2$$XGcP{;wQ@e!3lG7f60R8=jk-xo-ZDfO#fmx%ejin>zJ*<3k|NUpb zQo|kw$4upa)18oxg`xve&5ln)TUhXvsMNU&79yv3fgqq|0cQFSB>(nS%s?CvPWC1k z#{6+-yL~x0x?&Xq-K{$2C&J#*0%4AT8(GVKyDDvuaDtn{(EJCgA^ok*c?7~1T;v58 zufiN1T`^K<LI62Js|n6;6k zrStc)Z-h}6^RMmD{Q5Hj1=?Wb@n4SxZR>s~wtqOSKafYPWPIcJH-4g7=VD>$2vfH; z+7TNUV2-}$9k8|5G6-wLuC3%ZGcpx5MRU=?(H>z1!`^`-;D993yTo@kvUb7{h?P&s z8n8jy3<1=GkH>8HE#Ru16IxZJ&pVk|ni_#SG6=W>0*K_-JK)+kA;mUFcXP^N$Jn85 zHLtKOkR9+=?0xXf9Nn#{B`|4xJixe_S(<~3J7WN}-W|qn8_ZZ?_Kq;L0D_-8l=|0v z_(Pen_>CbG!l=K?H>1akeh4iPnAh-O+hA-v9}njKFC<3%yCeP~CTB<0e#=e1|36L6 z4o&|D;`NK091MGHjs9^l`Y|TQl*a@H0}k4sG&x%5&#R~#+1Ogc?153)(SNk^Isf&F z&i@N4I>xK~A&QPM8sE(Jud3*~#_hlK5q4@0&=)lKgjWD;hM;QMysxBb%%c(jqCID$s_+an< zI?2)9Jn;|mdoi*2Z;btwb0$A?ariTRf6KmKL|EGD>b{h9?Vnd zbl`st&cMEG2cq#60rIbh$q%q$KNiq|k&mC}IRE>`<$k|K{uSeL7*G1=vCjFpD{{@3Ve3;zjkJ0sin?Urgf%`8(Onf`xkALm9VqINOj_?)1%=3#UtoeRq%5&GS z{Z$y%UkJv;|97?UKRp5Q-?Z?*3XbU??E5bmZ~2kI{#|qUS84h;kfJU?C}H*fzXB}& z6d)7-4psk#>jEa+|MPsgf8U77@0I-*jF|9Y0$xAFnZqVBfBop{Pu{rk@9bdze#(wz z@Xu5B|6RW3|Ek{Y-vU>_%lA7>(=WJ}T zGSBqa!4~{o_n=q?|Ei1CH!tb$3NGN~6a3}T1=w>we#{DlneI2%{s&$Fa##QIANr_; z-@yxbd9e`5za73{cWM5aS@%=xIt)XoWuulIi!}W7&0spSaWq}`=-T8~9 z<$rYWE_PY_jg0;84@&#_81g@}f?^S`Qug*npbCZuyK%Ec*OdX2`{xdR`E4NikHW^n zzY9Tq-8O=a?4LSW=D&D>DEdCS%mHI^e|G<0t9Jf-~e zwF9jA{Pu?>emkR`yV$>Ov3mpa0Q!$=>@G@m{6$_kIE*-o=cF`{(8)fck^_s|HFJ%* zhnd&oC>2SYRd=g7Y7{up)qoHVPS#$$!cwn3?7Ejdw9$9)1z$KO7w%9?o2HND>DZ|2H2z?Ac64 z57IACTiIJuIs{0Tg(oFRU%#PNsN$aQc-xE72M3Q3{Sia_h76kR^7Z%5MZ$Yzj4yef zeA1yIxU5OvT(A&-s}5c)(=2ryniBX-wQnW%7 z&yOmTW8Xum@2K3%656Fnm0Ntarj2r%U0vsFSDtZUx(anP{AgwwVL+Ha*)CTq_JYCC zg<9f$$D-NCl5*Mxchd~yQ}Zlms)m}zwy9P`jUL!h$f2mT4YS=bL-*;yCxrBBk*!_F z&~ukG1?R3fZ_<8tv!8f$DLh}{QuIP7{2cQmI8$x`qQfyQRZl+UxAZ^{L0H3A{P*;D zJ05jO-)Sg^S+@Bh1Y;4t{f?lFp{hQ!(lZaQfsZvlvE7!A zhrqiM?gZj~$GU7CFe&4U(Kf-el`w*Y)0nxTJGl1ndoD{dKExUF?>|idJ{DmZqKq>F zvdK`;?7>oZXV6ywiT$e{are*<`PUOmyI|BA-2uUAg72F8NQwb|mbwswkBR#6-Ih&K zVEi{Z`1WGOc1Kt%j6ur)DT6%GN|I;_ziA8yb~)HL1yJp-#iOUmm-)((DD+?<4c7!K`pFnQ#=rZ{+?fdDV`vxARJ$P<~JQAzFt zqdJ=Umijj~3n)tf0U?BIuT$43Aw*y6P!U$*(H(TMf8P}U0})_&sXF*DFqFHw!%Cln zU{qonVZ7fAoD#Wrh^jMNK2bc1Za) zZ+qe7<2y!K6G;?=kP?kkYr96wM*<9NL)S$v=sS^lpk7fo`|(qX`Uk7GJgcTd#$AQx z7qa_OqqU>PPP>G!!^=x>lnGeaHY+Wy114?F1%~RB0+;|}V@HnT}Wljew zZ&t&M6==V=(>;OGv1x}T2)n56)qa=qyQg%eM^&lmw2*dFo+`)ZDsW>2tQBlaS>!|N z%$1{S0~^aiIk`*EQC?8wsXR~_uw*pF#ubPq%Nr%1S!S&qYpU;tGn?(BIV2nMN(NRv zX85+Fvh1 zs9m$ueCxt55pI=L6h8a-z>uuKI>Q9>uBQ7q0iL((=CFPz%93=roRuVtleg>g2Uum? zm+z{ciZ!=*v2pWQQe!sRyxcZb(i(;4r|Y8ebqlNU_4R$lN?>@Vi&^E0H%CcpA2IZO zO2LIFr+lpX)TMg#TF?D*)0n`R2aFP%TgCczpOZdh>Z*6;S-HHmdGekDx>5G|LJFmK z{CPTlrTYQ}EU#xjzApS?+OMrV5_`<+uE*+pU65dASpU)P+9WEd#e&HwAj67f3vBX{ z@BBOE>#(Z(@t-~tk0e!<9!T>1a`U+M`v=wbE)wmkmxbZSPcKb|vB!IG^teog%enM& zAJr4!uVz4|q-n+-T}tKJg8S2-bw1xZl^bW>qBLEW1hM5a!KubPO6{h^26}#An_rn$gK-4-zlw+9zi-Gq&0f&RNQB7y)s>n zq&pP+f*MY3ZwB-GqrQBh3gw7~GUxXpuLF5eW4KeUhsw2JrKpdLaJd=hIa>8BfKvIz zvm&K2ty3eu#nX3R1L>KyJAC}YbZ-u7%qhoVB){5zj2I%P`oPqWjnlk=2hZ2sM#JTU9!nDFPmK)w0asf(gIer|E1_AqIFXn7tyGXLrM$g}I@BOL@^k`xl8tEVzh zLhd@mqY{mTmUj$neeJ9=5=?69-!NSdv`f|A;!IQQQez!y8e*9r-=h-flc*f4RW0YlxH4eO@F?rqp0jX$o3OP=p_1f*oZKU&kFw8udpg=aA7)d1yQ*nLy}jLftenOh z+OdLrJ0KuJQD)nv=Ol&WyDj^K^Kqie9EKiY9*eKs!%ykuIS%;~v!cz~n&Za6)LkJ& z&xuI5&6P9mPoy5kUK^@f&pIe99aS;$iu*$S(}q~sREcP{AwgvTOL^u2XwY2!F}kGc z(xB_wUL_(|UzV(H?VDfDsgmO%N1he-NIbu48oZHf{ruCrFqVQ-PQ@AFvH>?rsmP#y zD%Hdok-72$ED;TZ0=q^SYYLE36?bMoCUp$l)$V8Dyy>;d{>hb6kgg9>4(~l;s-UVG zMU=Hayj_Ks;B~+o!L-B**D?Q#Ne3apMAwKRf^HZm^{a_+rJZf%{Tj+&1}0Yf6i~!;2`sjjsOr1(uBK zfIODOqm9R`xk=2WGnk5!JJy$<;{L9m1&~tC9m9SHO0;U|)Dd3Nt?aqpWlm%0kKFj%7C8U48 z=i2J4F`2}p<-|LOt?F*$R_ZU+1i~blJDTBBh1WGVff}mdDS# zl%}Z8uKD=hc?O~51rLgHHKz3oKSZ+FAfM00dw=r3^kk$()=)74k03CON24Gc{`%Ow zQseZ8(kYWyv&=T}Z4NwQ%XEh(ZxYkTU2VB6cc$ZXVjr<^AG1xYpdcJVK`6r4bD^J( z_xydqk{X@qsEMoy$Uy9oW^P#|85tet?t1Uz69f#Zka5eA?>vq{3SdwJY)-q>Cgt}t zAr4Hkg^1gF-;ilMS#ptduz#s<-m(7KLCDh700efmBTRPYLX?(Xs9EX0YpB#>+PbQddYU06uq<8+NSkGJLe71B+nnP)6~Yu7JKMaUr9`A~ zNPBpuEt1(&%sCVuB0-z;fEdy(=&cYp)2oc->nT5oOoJ(R9Dk8+!w;D1~Nn29L&vEtZyj1Cdr*b%>{kx z??Ki3n8y+4r0QK9$*|9z+uoxseU28IuD|}G)iv3F?o428=Gk@n1B+{;HEoaM^2q{b zAIl6-!9LeT&aYobs?CDk}Fn=?G?!w|~12s+`~K;i1QXBQE? zajr(AbtJOeC_fk2-~^(M!16{Ld0VJN#D4OgzD9o8^Qq`! z7mCGgi7gM`lEqgVt5)?|pSKqxUiHZ7I*n9CP4$e_ogNO#PDG5UR%=o#Yo*_ncVCR^ zKiX3tk)cOp!cx+-(ooZcal|TsZ+u1mcNjdQN%D1l*TRbTDf zk7W~WTJ|Lwym+csbdG2^OaA=5lUZ!%!%o#lCr+rNdgKVUo^OwLz!nnmMO{&MP3xmn z?hnvGIu`PGgtQh#NW{U;IO-Fyqz2Vd86F3+2ss+(ed)4b5ri7L z;63GZiInET=6w;*r;eW`E(ULxylr_8v9Wn4zFld3 zEG}ANYpbSUjag6s!I}OADVo)d7q9lkQ)N15?4M|<*MWpnutB}CE1ZW#r`v{kNfm* zOdwV!5W~(YFr}!_f`Wp3FTXyOtGl>&|Zp+ai+T&&?f)zV_fg@nI zoSnR%=sQ`urFCVXF%9P0vBVV1uJ+kJrEjXxLbzd#8;G>66xlc)#tJ@A2ixbPSb@jr z8so!?=LV-ur2?7^hsHS7G_JZ#aJmK@k$pFe_eCsEN%-1c^)V$7-8w7IR}_L-jEm5r z05!$x@AvmQT#qVAD2x-mB6mVAM5WBHKbm#v2NLiVp`-=7R?^yd;V}(g{kc)FbiC7h ze*8NVJ;;d(L~XVN6{>k0PB}8rsZCk5U&wCQO_$f+wrFU!gv5a3oyhL!o8-ZcJpM8t z`SCNb_fHdwB9nH1ft(ngj zSaH@m?vY_uAWScph^!T;*(MHG%jF)>a(*V+d{r><{ad@~N%~Uk5^4zcn=3^d`x(CD z>D$Yaed1RquDRk<>#Dn6CAvm(Au{0oZs&uifq5K=e1Q9vmOFC@BTDAv$>X{gJH*e! z-1GQ!$C||#RoC%Kwdu(R1f_M_IWILVnl5tWTOwYycj9BoeuxgYGN7{{NAo)pN-FzS zXi|&aq*jSa57S-DY|ixZqwJ2ni;Ia_OW!E;)&>5o%|>tU9a|Xj1H4 zoyioN@3G2q)O6F-Gj1*cPOACx>Va4a%Y)dN-U9I)W-*C&IgGW%458^fBR99Inwpvb z9mVUfp{WvJU57H1PErFXM9eGogro$>kTe%s9VWlqo&go zKsyjN<21i(w+Art;G>qWav3TIw8r3fK_?tv4bW`zAa?#fU%=1z9X<@-2Sx}B(9t>B zsz;odhJ$)&)@T{85w9{ZhgqA? zG$P-(laxabj_N;y`$`s{A4R!1%8Q2cb6DNo6-BQgY}4`ZV8K&8SsGIGhnW~TCpASDJ8ZdkKQD8gM)duy2W(H=rZ5UZ>a z43O%g-+3U5U`2n?Y4r*1nITyXkMyt6AjXPX!gU}E<0mTYRPyieG3>mW4YYPr8|bk) z3`G`uEqpaBw)#F1WHri?`wmHhpk=TX<@FrVod*m_W(!dh^YVho%7Ek*9gLF$Zh`ho zKv|nqe1`ACDPU)IBubSSv`#F`?>SL2>%2LB7E9#80vtA$;Ao$NV1`wA^c`G#VGcVF zlu-^kv~`SxT%Gl!x-|Ek5Bq=*Xg>+&`Y0C6wU46?xntBH909ErJrVgT$f2pVW_K(# zE9`Jp<>(=}M|Z-QBAL4T>qd7T@Hka5mYP+6tZk48^7x|(4Eq2MXx|CuDk%)+TBLb* zZbvso*g@-vwSIW)Y-P{lljW~UGK}{iPy~u62FwK|iLL$%>bng)#%|@ZPot(`v_$Hw z!_ae8HFekra6tQPQZQFIn5$#j$>AMKWvd2Svn`hP2@Uz@Psq=Fbth{GL}=_v@Ew7v zrP1I*9Y)LXpD}5x%T0UWPaAVRJ{|c9ZUqR zRZ;H6HkBlZj*$MZW+a%zmhU5yXdDQG4p)U;gYiY7ooKb8-U~=FiesE# zA0V{ls}u)wEh4hjZ^?KZxDRcVD-=QNhpo%bJ$_XEna4`yv4)wRYS8q}9C;IH1Y+~U z_p2UR?^yAQ4j{^FMH`n6#LY5uh--bb_uJ}|Nk!0Q&_aRA{iscO< zeTRlcbGf(oaCz&{jjDzhyXuGkl5)^PYGS_H(bkWOcLX;4wW}@A)wrRF+%2DJyjWpG z)Ex>=rMNS$f!hb^MXu}@l@F#5zkByM*bo&R<`LhOd*c%otJ3c*atC;Z{FhkGe6W2< zS_1*RodMAUJaxyH3#4X3DFv{|2m1-U)udWlX(8S*G-Y@XAme&HIw66x%}SFB#EAsN52Oa!5S^UM<4G03>6 z$~h)K_O6z+7>d6vFZSUqo2p_BMR{GiTgSN~qV^VjpF=WV@c2R2Gh5&13qif2=ZjbG zUI%A@$G(@Rzd_jgilQDmY8tk$>JQ`D71}HJx~{V7`CcQBX3Jt*JI_WYIhc-9V}O_@ zqIW6P6?Qucw)b+z!*yC26pRJg?GYJFJ{7vSc>8xK%V-5SpCe>^JCHyc>;W!6ScNpN zFb72o6KL3mW!RFjycq;MnWh4B*rJ8C$pjbc63xwc9Rjl;)m&Pl&gZ_znRe4`Yhs)^ zlACJr7z7ZNF$3g`OYOPr;*Z1i)m00c5*0<0wUsN8PWK|9*${+$$v5g0WTy8tw&0yCXq!qeY z$_1AB=(K0v1N_CP$Ikg<1>Mry^Ze5ITvVOd1b4B%r;(^29Z;bld=bN@dKmK_Eu|c<6)U^ezS(q$I_ab{*2^g+|!Hd(LQb4>&oa6Xn$&m_+%2 zF3%Mp;&PIG(pT*UGCvDFg&$3l-Vfc;>;&+xZ25B>hoU`JE-K9jd$SnCFY0rLmTKrS z^iR#xHlN)ldLU}A1K6YI5?s929u)=yUJ;@DO+B!WgdNc~D9F9MK!S@PN~abT>60m$ z2&83`O?yguzEqIUEdagzL4bZPKK1G{)}QxYR|{XBUaCiW3adKxoOGf)3`Q}?;LE>$ z*nc?F9cAQnL1g;!sk)^S;7Q%0O8wL*P7YmpF&#*Ydm8`yl>5V@8!v%>&Tx3p#0_~20kmhtdQ%*u1kXbQGdb*xhW!K>;m>~ZgtnJ5_s$U+$=mOAGU9hJz zwRuAhSSBe6Ox2RV0`P5_Bfq?L^SC`4Cf%%Jh)xb0z=Mm?iEJ1UXhQcJdZ113WtugA zpJni^#85^K_!mpS@?43}g`Wa(G_7WV;UQ%K^g40|o!+Utv`herXcDSeNpiI(I@6NftwA#pEN|4!Y5$ zdyd|?A38MX;p@qy26pZWBWH3L3?I8DUUY2H1})s8V}~=*KnNN@i^{kmEg(|)S~t9y z20g*InT}90;IVoFCG0fM&{OuKD$*2aK(3Ihpx31fU=YY0>!+t1>^cM}NUk0SIw4op zYJJj=>ORHYA;v*w&;=F6^-R#k1c8X5t>zl1%s^#a5(|N(_0+} z@b@51nT3~hm4uK%~py|0M##e=oRFX6h*V!Jj4Fg>S+>z1- zT`U4(6qKFIoO*O>sA^dvBa);(G7===iYTQ!(6h>ngy};m0gvq$bEZ$%^nsB}O8~uF zNK4^nVJ>M(3J+xA_cf`x@V0w#A_+ATIZ{&M>YH;3XsVp`eXvt(L^FMrTucZ@uQC8n zTQl0o(@pVNR0FJjX^MOx@A{dpqR5wst&2K)B-}O%HgbkYQ#4C2>r8YN&UO!U)Yl`* zP8o7<%K;eLr_Py)!~rXlPcM?{O>0}1S-s0VoSo%Y$Qyt!AX9NyGW~Uc&M{7tQcv|; z+|XOYnE?SZ9l#;1Rd{F7WqTKh$0fY``lW6jEUPvxCw2f4L1zl>(b?D}uc^x)(mYSI ztLOODX87lK#_*e)cBLLmgl0@!=ah4JPXnDWkA}An5e;uZW|fpwg|^ItH5gFaX$Z+Y{nd66V>?h8&U+A>cU1<>J^V}L!OTHqdyIA|~-bScZD*%mjcFv)* z=dbdfh>i)I9&N4PoUf5vof&*JXc+WhHq)bQ(n@z_>r-fhW|n>7k$3SD;V1YEZq8P! zz0VesLy}Qak+AsqGy?UQT|IcHYSxAn@_0N=z`93jex!Z^HOssm*=DMc#qc3%@pbe5 zwj8Hb@&-O7AwM4&-t&VHW6$)!#idi%oh@YOU5;LRrYn!$Hr|b|Ze#*RyK&miGh*LJ zYCT7-pY|3N6#sG@Itk;J5dSV-1PU+zytn?efXBwUOGZ+t*~)pj%HF}W!~K!CCjFr@ z6YYM5W-9@Qp?(OnHd;R_EyWxF)|$Nz7~_>}Z(VI+iA;dAesCYR)!pbL7WMj=*y|>9 zil)2xnMO&W*KFP;R8Bt{^-wDIGNKsOPfy==_rmDg0?U?>{GiyxbIp%G%(lDnWlyw( zpVsD%S^KO{BVkkIQM@+!X{MUCL!rukVMw`Bd52tm_kgfMFuiD^^Ws%L z)T()s#@xqOi|F*G=Z&CHgqQxQ+v}v%5pXBvwCV$JQtlR)HvogT)n1;Ex8K!0J(=iZ zLx%xpNbkV8>EcbHVJ)4Ig-H{x_@a=i07~9#Q7-21?h@3O6A>BUX`$0>GV%xO4*-Y= z(^+?g_#u{0(?w264VyuyQbY_N>sG#%JCv=#XoF1fI-0DCdKS#U&Qh%+5KOg?^jLUo zW~mnRbC(J?mtMO?IY>i}T$3KF=R8|{b78D@3!)fI-<34lrv31XPKp;rr}x6dd>=~C z9$Mz1wt0txFVWnkut0isaH$`Oxbsf8K*6g&TZ%(7{bc?C^15Asgi~+eCmoNDCpJmu z>v=hrU(8$)?QGTYK;$DGJtvz(Av~%N#ZRH4BkHv`8PnP`A_D3ng@a#g#(OEvk2IJp z%%D?{<^3Lo?%JPuk+;&f7KJ}IZ7>ZhT=2JPA$Edx%mrbS;ordLdOd1(t z=4+qjcy@4GI@U~mcrMFta`4OgQ;8tXUiEnIltpox^{68vGbN5=51QZYSvn?myz!uW zn@HnXx5;~U?oZizUWOMVQev5GwTAq~@;yc}Tuk3Tu9{zN_Z|r+;Z%6igNOhaA^%>t z`gis;{#0Z48@pb}uRNisB=K&Ixy0<_BM24)o#FAC3>`}7gELM_(-xb_lim4^E97Oy zg*nxw^3mxl(6Y;OrDVp*=FsD8Y|8x{+Vv~1Ap6Q69h-kIGF*5F4GR*6x(9J>dPOGl z4EOiIBWqvZyyX`BCN;i8F?wrH5N&Dm_9PXOwAab}si0nweWYDdb%tI~mn7@== zHa;KT=BHNL|4>}LHCh*?Hkca8Szlww#j^$V#7#Nz0w9`%xdXgTZo!A^E0-l=SiDc^ znsPaau`8lcMI0ioN%Ag*$sVF69ua0Rr6HS{RtK}ehKX2G74|kp&dO}k@ zR+*kZuzxT09=USv!%rJ|D(_fjCL6!(GkJS4#d&pPeoilo&g7^N{Mvr62^&!fZ`>67 z&#d-6LBa}M;jBrY&$`swWH69X_?-D-L04Gd;mQ)~1%IR9TDZFpmposR^!~AW}E&$$U6pKlUXFK#}HK5asm;NQELO zm#dYFZ{DvN!;&vWzI@J zBnf&TqS)@eS~wT8{(%lbzEX|UGF~?SbVnHyX($XQU%Xz}EtpzA2?$(tPnX*U8P?{( zz38Ag*>!rs#(!3Npqq~m?D?QzuP~vqnIl_Y8NfP_b$71~1Ap9vEi6e%N@8&qVYeRZ~&phG>PW9&xA-G=&Th2~2CpmJgNG!WT3lH%m8UK`2( zjBDNY4gf4P=oqRTYJ20>u7{vD9859f^cL_g+sJjan+=f6eQ;l-Omt*VSGI-kTvAWz zlo-G+K!huA+9xT~^r#uBB(1qnmouoDkd(CAJOR+19zqCS*2|%)jx;ZxGK+P)_gusE zbrlm+q~dp9r_j8f=f@5qh0sF(iTi+WNlYGd3Icb1z(K z=o*Qx_C1MyiCDV$mco_jSYy<|#`2Tf8LO{!wEd|X%f*!3^4xn1x|wIyw6Bzipaf}z z$45eM`ii%&iw<)HBA$9%aM6y}&kb}Cc(-*HsGG-Zx2eUfj#9JmXhU+}N4iwg4Y>xm z)6te6@?1-j%K5U6NV@n6ubK}P5SZr~M*{VYb~6Vd|0%tmjy+?hZ#qU?7D2>)lD$#z z09}0+oiMFL1z|}mDPC&`k!su-*v8w(K&R-okHC?3pE`1@T_3uo98+s8$bXJOalT$D z4usQ1Hj|DrKiv0QXq>&5?~5-ui)v%3sC_Qpr}V7-SW^rEmeI^>%l{?vY4O}F+)3WI zLfODIZPb>dZ=ZA8U=200B&Dqtn!+A$I(g_!2 zH)Gax7zjJ_J@fVNP!%#3u#YSP2w>3goWwRfqI+3bP+YiM(2B#QU!U#xamC`phoLL; zHdE_T4tyWi4JvYR{HT`h%K2|`Y&4ELZ9hElsfz71owjSLcEONrw?j61cQjUfhOC5H zymywl*VTvEOasa1QhA!`^}6fk-r;sF%SMwnX&?*T;yv1H{o2dB z$+|rSVhatpn{OuVEnf$8kx<@8ukb)7l;no0phxjIxTIbr~f z?OfN&64-+FF(h=g^aUDjKRo05f$)sFq2Qw^MfY@#{D$JS=^JBjE)88Hrt#8Q4oKc- z_1-Y&+(CLVYyF@mu+Lx`%IPw(8@%W2nVXuk@g#+5Y9Bn3g78GE%}Q&=llG0J;v*2X zH-s_XJuho@#)@N((KN;r>br(5Deun>T$BBjc6dM#&YDo208dNu!cB$xjH=Lrzy4B18GPyCaC~Nbu+RDbY`L=yCG_lCfi^8zVg_zG?osZzG z>049d2no9Sltvq;33(??ca*k&EIcf);r?F43p!OKS?w7{+bTJ;VQ<8nY{xIY4uGG; z8y>FnrbzB&`0EG!<_Cvs?98I;$786>Mje(WY25NFBX`R%K=|Ry$G{QW+m0c|r3Hze z?rN=6md#~xu9l&5O)t?1&K&kd&Q9Zw_PT{S5-0a$hU+)$js;NSBNWLyeY&B|B;15s zwC!lw(B?tuQV#DeH|_nvvZ$=`C?cYf+9oo^pk@X3{J!l|uM|j0E5sUaLIPAAC@LxO zF>FPxB~=FWxi9+%cAZD^8dshhPH?aqYp87kK+EpZuthT4e6R%hyB%^_&I|E(gc`&= zWw76z3bd>qNK5{}QT+`;AVYgPuX3o&D=9rXG!iCYj}4iv19^?I%U#EJp3tG#l-wtEo7A&9CA| zF_rM-20RZ%$45czFX)C0*g^0%F1`z>maYw5PD)cNoBZ#<9^ z219QOz|1o_Il{eM2jH135jRSifmEC2E;->h-u`#9KB?x!FGYO%N7+mjmAzwVC=Tr+v z?Tbz!$}Y+(a`U?B=5xOdywgcDPHUeD|dM{djP zy~&l9m%H601>nD$MkwTZ@8P4@>yvvRDUY=K;3xp|_jBUXBH8Daac`+^?_wYc1zs{I zUn>R0^?nb%rL7gu*|_$r1{A%?dL(FXQEw&XqPbLM@hh*ih{fJWV640!P1%;eyr>OL z?jcPb&T%vQu+U*h;tEwLrOXKA+^cTr`K+6zWVU)e6nYsX2XBGWiUxGW)dW{#ZW7R4 zUvtG%Zr(Y)9tJAIFAO}Zw@V$L&-G?ha85mV^?8<=hjhGYSys2mK53(D@I|OYb;zja zi6?Aq)Douk@eo78#zIh=f@nPG-G;2zS>#N`?@)FxhP|jyIske1er&Rfe*q4h)}1Oe zW{NA&2hZub!=^8#o9iMIrY}IxmW62)WnWwP-1@4Z>g~Hms#`@H+M8EJUI3F$js_@C zcD|55allhHGgDtzT({t&T%fnM(vefl0nd)(rzMt|sz>ytJdv-1Q9uOX2g19Q{YkS_ zlnyzOuHMLRkJg-MuT8!kaZZJJ52>EIpQrnsE?RiCjeqvCh0-(U~q;JprnWo_gfihTUbqBpu~dt`=v}#<#+1^s@M*;bU3O zw`(8)5~_>5v^ZZ^?clZ(8%ZTztltvuk2Tg`JVa=xiNGUG)7U;eTjhm7ick-3o^rt*?dY2JDJAN*4`X zwI)I%f^H!w55lz_uihh{wf>OfOjyixZ z=Y!X#W%_A1z6A^YMTvU_iPEk#i_@mVy(*=ZrSrzqAF~1*KjrUT;;#%-=FlMQFG6k% zMx%p5K~rFt)S2U1+&mDQnZe|F+J!H(QYXha=}dQ8z8&i%fNCenh-)JgGPw&szAsL4 zoclOG-K3qURz(7~x7z}lLYfk^tFBwm(4nwljwAekM($3wO^LQrZKA|SP?RS?3@eP( zkdFOSk+X0am-;gGfirtp7SGL>U*9(}C*D|Vt{kN}m>`7m`bhNp=8X+n^UZi~n*8O9 zh6*V?Qb9)rcv^^4o%aU1zDDZYUz5#ri{vz@>wJ12xffcmwaI9+zCapd%>lVomOe)WQFGx`JgCA@~q$G($;8_sr53(mN9Scu8$wj zo!w>w)lAjAmkf?RJ3%+r?ImZ7qCA8uBrCV>JjpM<-rM<9tbNIPKnrYQnF^FTqW9I` zjDOEBzULslIM;G(u`Cp|^pfn(J|JiHr5%T#CKk)*t?j>*@a5x-7WHzdxbsK9IoBs} zha~Cnic@LKn;)c|)C`XDeVTgyICD}x{%~CuDDU}Hs(n*A;TnPTjrVrK&ND~zCZQnq z->XMgz8pAP8FzM2f5;#8N<}4>F8(2k-=MTra~-9tiMk29n%TS%SA1^smdV?mVbk?H zFY<3{#P^`&?NYMdYkOy9Drq`Imu-6|B^MPzA4YqYEm*M|JT9=-S@BOA^1rIyH=0sf zR9`W$O%3XEH9C!X_6vJv#n!7^}G_Hj$?6W!;ynN`p(U=Ov}L` zPp96s_mx5}32kiV_i7B^4^<9n%L;1CFlf!V(3){29SO206nCum;93#zkwCP(#q=_Z z(!(b=XCy>DJ|*jGlYE;gdfLMY1E{OyPHD;Y+^N*|lO)y`2I>r zf$@S_AzmeIb0&uXgL7VnU{5L_)FeJ`x1NKu+|-|uSu&pUNZaqSxQ2h z;u`BBTzFc2plns9S_~DCLnT*sg5^XBr@NO96Tz1;;oOXL%8$w~)BRRl@F%I>a+JJH zm7i&hW3h>!;si0gBlb?0wI)R8W>Q|0uywmURtYXvO>TMG0z4@Z1KjRLkO#fyxd#{D z7+`0~ICb*^eDZ-D09c?qsk7k>z^|m%R`4Qjx$&$xX=Vv|`IlT-6%H@(E(i5d8nk_O zv9QvQ$^!K#yAI(+5uKxhwj#Q^3#7Rg_oSYhcu+;hT#}6j$qlOEs(#SFOkKnZK&SX1 zI|;IP0Xnrg2KXhXDoXAFn4Np}y4wZh&=nQ*?u5&rP-(00c}qc3TVH-_B#6?OT84nC zFdT_9rhPW-9E>~hvrE+m5*C~8Le-NghX(1YUU(LJnwz`vhYjL zP2aT8b_gW6*egX~`248~McVc2e~^QgFPgAnJV7@LbJI#t$?EE^pDaP14h`jU|M z%X45B5C1cZzX>mt^ykaYve({r&!ygfT>%2%B9jz^1?aOrJO!WU$kV%Iw3_Nc$p()% za7NA>N)SOdv>oIEeGKp<*a9QGwYf+wGv#Ks)XJn}ii;${aSMaeyVBZw>TF+9!Kdw| zq~kMvQ=g@pR^I`n3nx<-F_kYN8Wdz}=Ka{V*XfcJ*EDgB0RSvNV8a(t{BkbFqpQ|a zA}eadpyy^_DL7Ij3E-SOKEHZICRwrT=y#Fhi4pNYPLwoRdON??`3U0Ecf((98VWD? zi@%X%bpLE0e#+aI+B!ys*$33~vCET^)buGV_LegMFjdDbhRcu&#rum-t($3mL9xNx zYm^D)*WPO(d_Yqh7IYHS^@xd$%)WrO{%N=gUT;YtL-A0~mrzTr0NPgJ36NZU+J`ya-q>{z%jONq z0*h3lBe=cbI&rNq$IW8faZlzhI&Fy*pG6i2R~esNI5Qf3u1F7MV{x`$bP(g9>kjMU z25uV&;jok_h}!9gFHe#$vO&(`f1DqUn5Vu{6T(oV7j)F_VqaHbJSBMWle5F+|-MKntf(zwrPT`UTa z4BR!c79I0bW&jHpwDoRC#@=iIekq@SXvEHU5@5;Z$ip)#RvC^F_Q^{kaD$6!+oFdv z<*H0ih7sC^Q{JYX=wuJ=YNJXe1sFR}f<&@qX z{ALpHo>C^gTcs5@{OoeXjjSi>9s{`kY$UcYr;L{Wx-5?GyPZ{Xb}L(p@p#F_SS`|N zafex-M0)X)?23x*dDZR4Cmu17(?ys2m~tY;^Y7e=;iHpd6aw5c8K#}cu^@t z$6>RCrbciw$<25V^Ce_>Z|5M;~A`K$4Be7kS>D z+aH+N0-&G;={;dC=AUz5_R4)lVTcMFE3^6&kFN#j7qW6|bLTGxZ-Ed6HF)Lg3VZuY zVBtr#jHr4?K(O}%xA^P20f=Eq^$(rfr21EYFdiO1jZ}Z zaT5FwycO{Eso8ZWJgA%v0323dn_LWi>cKc`#V3_4H$GZ?KJr1&_g+IVTnbbAp3;^A z^5M?RhfoG$&-{yzwtcoflpA`J?LAd9>^w@<%8& zNTOl!=J@wYM2A(=2s_y;)_YZLx&gT6G(g99C7h=NNHrfXJP$8`O3T+ieVVzF#C+2d zeS}97)TcIf2MA3#5%n@9e1HM-@EZf`)=VD#wWs-n4p02nhE6_sIHe zd7nm7klI@k*nTOE1n^*fHGoK~Aj3U@2+nRfiI+z_E1n-2TMe%51=R{dj4f!K*P{fS z_+x%lwA1au2XD_-Uh<+v1M(NBuFOC64x(YgD{1WdeSO0c;q0muK#IhT#%q16cM@T) z_|!n=Oe|cuYT0N3?u_@<1_d;W8%NslNwkXg$Eq1(;M1Iv0T_oFN2w1cwXy!*aD{lV#l7~)~im`n|{%76qdtU}c z-WM`x*a2k*ZvkS6x`J3mELnkGI(4hS_ z5+2u<^6J1tfC&Fkc;b$aW@@(ig)bL4{iwE51kv^tpPuZ>{3zk+Am%G+I5Q48Klzhnl z$%a5MTG1dt(Y9#;bQl@`;`g;ElVoV7f<0=3Lx4{((DST zFG$k5(OybCxPIFXqvzCIy~sfU{Ey|@IE9Lbm(hKAoBm% zd+VsE!>xZ*5R{M<7^FKC8A3`@TIoh&=#uU(rBhk~L0UjsI;D{kK{|&9iJ_bOjOU#7 zo_DQx-QWG=U3cBP*6|OPFf&i=XYc)d_WpkMp52fq-3Q>wN74_Zb$<0j0A0_Vz0{==59ZT0Dmq{7;#g6XPOHAA~$B zkN!-lHMndV2ayKyMW!|#?JX>vL0#*X);|SSApL|UhQn;Nc$_WG;!}|9X)lKSPvd^X zbnxMKAjiiG5~&{@wa@K;o~>pSnpI7c>o=IFY{mZ zIv5VW(e3~8ng4+|erOT^ZY&r}D`eU9%=|0Th$ZR4SY*QE0LnOnW%ha90yeodeR8k1 z*d^%jP_BWcQ)R9&Rcmh$hN`8%UxPFnW8`OeOKpfzYOw%~+Ruz%F~q)`4EQFoX8>j7 zjF{-7;q#~5=}Gx|iubqL1Cu%w@@B(IO}VsT@hNiJPNdZT*7?{0#w|nT8;A6$zm(rQ zApZo&@Q-`EFlgQD_R`Q|E%ZwJq^hT4lB9SGdFor>54WB z7|61wzZdx*{iQ(PL5;(5n2lDCIR@qO4*yjZG=24ojIA&Rdbl>2;2UnScY%ZzIMQZw z_!oE^vEi=i8S|>3f}O#j?^k?T=agub*Ev&@GX7)qovEq0~jp(;XZ>kL>FhM3$ z^EO}x@Q3Nj@O%1tceEcOmm~=`#_;CrJHW#JXJ5V7&|V}xzU>zK%YODhIzL{@EBJuq z9ML!9u84LGNRtfmz`cQnPA|(p!Qcl>egaQ)enn}!o>Zh%8GuyjRZJ++xF9s&4EKOj zv_u~wHZ~=LF)%XrkI)yaj1_{?ASNl*?44k_aZ#X9M32>b8JojP$sOrHW1{qxp-&PA zD}$`{POD*d@4iNZEE99WaIb%XVqe9I>C0ambp(u&ds-8lb=#N5(wLt`YAbuXUf!e& z>=oo)cc+}D;tnOm`6=?4|A>XO}S+!KHWbZJh0Qn<{ zKvV%|*>IuyJeT<IV6MFTpeieXXVm#C#%3K)#vsZlf0edCXey^_urmd$;(;vcYn;BZkMMo zMEj6gDfrhe29&DCXBrGkzfz^1-z|((Z_pd_N7g6>e*16TGx@i+Kxc$6S{zNv%pr|# zl!*zV3b~<(AsfNMU7pGnZwG5BMma)5*m1+toE|9^s+zVXySZ>9a$% z#eS^yZ?D2k0`tsj;a9%49w^?$y?%sm*IO|QT?#*%pwv3#)?eZ4HZuPMq&6bld45L$A zOh2M^t@WLrBLTKvO?Utv`WSS!lvS^{PA@a%6b&>-;SfEZ_j|7q`!Um;v9nd?N!!)4 ze>kx*)Bz>#d1|hIl{mev#Ye>mdRlJV<@7p@gMH2`uoxQk1nRcSPvh-IjqAPqbb9fvL%)8K0C2;w@7&H0*id6- zZNDLo1xn0$Hq&*XpxjA)zfIX`ZGb&R{W1F%7udQss{_}06x%{$Y^bwt_>UR@!46D^ z*)gwJ z-TTq;O;Qo5_+|u1pwJx95IhEN(I35j*Wju?8+hHL%6nUd$ESe)Se>oCqmieiB%x#4 z^D5};tJytMh$Vn7&xpUg&gH<{J2A8cPXuwP_}K@PgzM-({NXxOOHgJbNxpM9T3)A3 z3Mz;24gD@ezed?T>e7H1{gGz_4G%tI{adw^BM|qisfz16E~!|F>}s+R#L&;18>Go< zsQ#1!k6WquSJV%`fl4Ye>@VPfXaAw6r(E0CzVTj0NBh;7?xh%ivzZ@R3`u_~v7z4^ z&4|^%*l4XF2(q-xjYL*U*f!~Wf^!{>Q2WsraJOBd50&2Q-}h^K$ujU+RJMFrU&mjD zdS`1>ZLNez$gIRl2r>6aFy<6vct|f+zD3V<`hJzM38fT%V(98oVLd@>Y+6ify1-C(RmwWt_}z7I)Jr z8mib~d&JD4uriu!nYLzB@RwTN^{q!s!F=0m>SZ4@pEH)ZHx4KrBU@*J`e)=u6nFi< zMF$ul8%C&ZK3iT{`B@MgizpWmh~nf(lWFS#gLqY)fx@7#X==^pep`u6^E+IkzmX4G zP?&)R5W@KDZ%24h{RlwWh+G~f)JMn4J9Mxwt9kxgNI)hUzrc}sHK_f6%~#+Bs@(tc z@&AbK|HEo?Jb!C9%`#PKmcj)wu(|G@GS)_YS)kM((|Q|^8jLNs1ZQm5NeRw@EUs{P zNif%d!oHve_Zjev%Wh|O)_JX9Tr2dv@HjutF7Dha*J=D~&sE}nd#Z4ssP6eqLA8@n zeWhxprONGt)Hvy#qW3Hg}vSSGF!#Q?zq{C9eXac2&*=@yzWcm%LKGW5g%DB zA{Nq-8h(MOu;`>p{QZl*rT*A&$(hMdabmaxSo_WvrkqzW9-S}7TJ)PujFp@DZ!A>S zr)!meKJxLl$=|uIv&?F`vLs(#s4^S2yDDtqVDorE!0J09-Xq0Zq*t~b+|NU6>iss>kFf?t#{{2es zX!in1(;{kpYkz{%-trCuyKapGpOx3S#O*b+n#_X)`!&DEJ-F1#DftR1#%6$WtofAj zrlLmL_WA(6@-r>zL=C{4qdy`)uEx^#G(LF`UX5LBafuGjCZ4kb+_T^~KMDVvy`()# zum0B!e|KLQT0HAmdsS~1iD9j2cp3+j2tg7{ zR1dd`jUTIk4vk!2m7tqrmGwuDc*Q}tP$I6FKU0-2L1R|Xz%@kpN&W+ewA=19Z742f zBDj%~Yhp!N`R25Tx!w=(x|n==%yqe3mg|9tH>y4!UTe+WJ5=hQZ*I&p{C;TGpWM2d zm0yzQT?>gaP?I@$?s*X8@@Gs{l|d>5&sTTXR$WWer;+xn0%)q))z!{~-sI<0=XvcV z;AH6U@U3dWQ&-c!S{s%j!Y#R&U0fRP!y8;+u88ZeU8j6cc)S~fJ|c;g{7>w>QMkKCrlp!e;>B*N)9n(rfMze}e- z>~fc`((uP6*KTl&>fX)$m9=h#+;+31)&0P`l}_Iw2MF>qxuor~l$Hd-sB#vPvP~0M zf`G|FT!8^OeQ+csr4nL57vAXh!}*ji*?XoY<$i|GW0JuJr!-JS$|jfmsgfQ5!XH=9 z`YGekLE;uD%mMd*Qy99XU|-h;P@0E)d@oPN%P_eSmm{-^EFOmGtuKhQ`iv+w5HEGfpDe&lnH%C9<&zS8^2DFcXIl;Ox6mcL!_xHb)E0cAGwO*?9P$du>8N zjbQ1AH!_4U3Slov_+E(#8rxG{A@d_Br(>qjfu%|PUQ2%Wg3Q5%?=|$msfqK&zqkNn zqp?4-q5g^77V%~irJuTqDVIRE+4d3R*Hx%6Q2}|up{czlM(P++_R-s`9=5>M{$OV( ze!s5A%Ub7!1{U!NdDl1-a&fA8rW?%8(f)wLTQ}0~G|R?YXED?ItKN>%Y9uG++3t~K zLtbktA}(rkTqA>uM`ufT-%!4xU@Mc^KxC8K&se@@FE1L9U<-9_w~Hy4&<{uV+p*UQ z=6#?e+r0ocNZX}Mr4A^S@bNTCtreK({dpa8zImGtxJ^^*>QAco>>npVTeO*QyczX# z9zRgdYWru;=3gy}A8lA|JvcA>r7Aazp`#_uppvZqV8~Z)1+XFLD_Ox~-cVySA*)pwYxh;}*CwH~!x!@xr`v0DPwBK6nwY^qugvmPeGo2PK) z&EEx8QNG?q6`njdN^3r*HrpZ7u2R8bCA!s0wYP+@jf-SAH$6W=0rlFXR8I46KCO$+ zQ`KgXiKW^qZ{}QUg?^s*E$*Zx`w6Kf`#+b_IrdbXr~IjOb9HV}U*&3Tb!}siqrElp z+Cbh@spbX|O2n=y|7Lz(jB0mI#_7hNhMZsE>&%MZ`O2qSv;L4{)#{-^C+|ZN)#lx& z-Mpw;FjFGf7k}#c!gn^8hf0r{9V#lCelK;6i}LoSHG944`7U8PbXHNh)ep@xPI}B` zxyGh)x^!*&NAO_g^oi-Rl69_WF!t&Xo$5TPVOYGNJuFAm|GH9<>Bw8ftC@S;Be^ri z|0Zr^ScsAGUNfCRQ>Md-Cd|=Mwbt=r{d%0tyWzAj7vbOeZq*qBk^o>|a@my0>_h1)SENmF5aKEq?xbYm`(m*KZB8xGIdoB7Vv= zEc{cFYU3bx?2EtkzCu}hjs5x%Pq&*@b0zV4x3i2y>fq)HMuxkacGpj&?f6&NkO2pvlsp-rmYL$O+XKpU(TPdtUe$RMA`bS)7~< z_`0uss0op`BF^w|T;3I4DN?1}wosHPIapm>-j1}AOIbdvKje2`#p23Ud}n-qXzWp2 z>JUz>hU>p{;p%x{yc1O@o59any3ibX@-EcqWIyY`G^kGVRk|O3fq>*xt zq#yAe(@t-3Ab2#I|DxlK#z@TzVM%wlWZNgzHt+IztQXI>C$yVw?`Jgogp;lQ$oB2H zID6{NYjR61B+O|DClZJqhQZEwtVvv~#?g{q5v91&8Jk7;Pcbo|NWp_G=d;vIRtD8D zN_5=fn>9*c4?aCVNq!qp2$$-570h9Ffcj^$LM4JG`vK`BIxzT-_gTU#eb|n+>czaS zt8%vy#6@yh^r!ac1l;Cr{m_bX|1w>YxdvRZm_PL#o?EiH@^L!THKv;sY1tyv5rQ^>KM=eLZq>lw^57?(V@tu;J35 zKWt^u&t2h6H#e8xh{U_2GGb(6sSJO7Q~A}MCBYO#nh8gzzCTX0XqncrgtNW;{n%Rd zgKUBT_f4}a_LH9I7vnzX9F|sX{*!#eAER-&^S?9N3FY>sW7bCyhw2^>vFJ44;~h*5 z=Sg7t{HkB`VD;#?T7iSdp)?VT0gABsK>SYVSn<2!0!4U1dg~8^TWrq7w$$HsjyDuc zJF`!3sWLp4jyLIb*3NPAw#OyXR7^}WlS!a<250xLi?pN;TsH9s`yMgA`JL=Z^TvHE zsyc!BrFXD)93my%<9dZNIff$U!IZOMTYSKBmX>0A+s}Kh6^mn!7!*F;i?y2-J3TNn zHU7hOOh%(rs9NW?_RQ_6{#%#VRgcOLT-9@tGO;uVei-CMwbrqdj?3^dD#zw=tJ!83 z0ux?4>65p?KkH37uQan6ba2QCHp}Ep$2*4@BUgD|TX9*u$TJ^&&!+E{KGmp&#dGX^ zB7RNP?BLRG?sI*B?tFP1XX2jD@$#muSD#KYQ)g2(r$5o^S6Mbj{#)k+ly~oW>_wV9 zPl){GGb6NOsp)N#zv6r9e1p{qjOE|ArG;^bT+7{rg&+!fTj{?|U!aj@a*3GvkRdXH z5rtO6blnA_bt+99wm2VuJr+WUue`^vXBe`ZVlils{lJMG$!C*jEVcV*YJX#s`-n;u zrM}X*ww+V)QpmvQVGkJFo79!YKUBtox0eM(=6y#1n*^Fs~aS+nFt zrsG47a<%)3PgF_m=l$C@rkrzG%Dp$~IvpLqYf)1tT2G@w7h1Z`M4ID7Tpv>iyHkZK zWyYCc5=g8xVJJ0mOdj9$7hrXtt&z{yGdYZr4^X|b) zZ_Mi_S}$ptE{8>Pd~+^(KQQ6cPzwd2Z;zYErYMsB5?GWD-uQUekH{cT{iTb|EdD9A zX!q=IYsAHlpx#U?dAOeb>TkWSZ9eHH{tC{|5`qkhA2`}>&?C!DuSi+H2KD?PTBuam zKqHETh#TVf9Cr2JG@jaJSyhqo`)p(r5y%rmC5Ixp3+!@yx=<&pwu#a|<@DdSTK706 zvVH1jfrY7|-N*gYp5Z&=u=#7j!O2OyDu3r^zN?mJyOf<=(jm76xIj;UdANW-D zDgpv$=!)8uV+?-o+TUlLM#lFhnpue|%dg`Vzvop55p=%U`<7Smnq{-J+jWhtDFrGq)XgCZX%mJMyHC$d=3wU^d#7j>#awoY_77M+>V3|s zS?_(~cN_e0n{8eazcFr0L&zi>#y+P5-_OE6;8t<(Ezg*&*kS4Hi#DS7>fdgZBy@P^ zL8};_tCT{(Q*^&+fQ-M6+*mR;&`5$ggftD9KPE7L*)$2ncJ52c8I##$+irs{?0iTJ zMRH$aut!{(&1+WJS29tO>gR4;88!IFKy{-}g0VpaNMgzBP(DH_2!XaoROD0~?L~62 zy6pWRUM*YiYGsyBRUbdRIl{EXT9Y^j0gHn%J~W6Up278vi2*z@gnxuKC{DzAonAAY z|A5;%93uYtwexj8=k2?4=Qk6IUw+nEy?=-r%6kmGC4E>QTsYbI9#hZ8SjJ~|&LRi`>FgYiT_T>Xavx_vKmbkg1z zOo(6JNV+MI@`g5>4=2$jXGPhMVhvLZwGBvwM*Cy$=-hK8);2`6uWxHT8_JNvf@tVg z^VazKtn)S`vxhHIlpyA6r7~wJrr%bs_LL`id_PM>6s~GPy~nUox(lL|9%V{5Ss3}e znSL0H*u=9Vg39YkNW}Q8L*?ymJq)>k@l?bu+TOGx7|$rsMO`?(1uBJiIIQ><0}z_7O&NetPfr zcFze4Z?x1izTk_uEpc0@w7izno2(}m;4J0dF5tAqRwHbfynD^6<1f8%|l)ku#W;I zYLy0gJDV9VNm$R(-f}1@9}~@Hu>eR4V8^cZ-&C4Cp|6pnxTRb)u`{JLEjjB!ft$N%H43NGW-I@#i2<{N;obeI^rm_?#$8g(cJslmC-fo}T zzKfnNE_HZ!C*Y3e$9^A7)2eXzenCU!aenZqRIi!E|3$Os_H(w^-@c!Drq~Ut6q%x% z4(xM>#QuqVnuc2HNuY(|vN=t0d76Qv*7RlKGu_28CGG1lI?4Q*T6^WD#fNR?XcSh? zETu%x-5o?V^;-ZEAdNVK@eRdkz);4FX3=syDt6Z>b3ACaA`Hfrw#U(zYj?$xfnDuf zU;8*-wxj3vB|dW0(~pLC)i4jgJ5x!gkbTdOdGS=wh6pO|>@qM}G4e&Jl?GZ}!U%gt z{*07oCUgLY_Rqoj0s0=SrIQ)YU8v5B`0(CQ$5`|I_p~xG98zIkbKco6%I3SR9w~Z+ z%~F0mT;Z($WXP=8dXyn3icwjez$DJlOTrATTp7Gi3;Ra#oFYjVZG|o3%RfxU+I}84 z{!AZ3e>-2kJ^%hKS!hI*hvViMXV^pPDm-e5qD((5WAo@SRu-7#3csJ725Jo<6nS%1 zZ;6|)r!uvj5jidNojqIOGWWSK8E!ut`__}LiZBKb^DIZQgRR~bHfb~;^{GJTVz+&Q z>OU5ZiRVA&(XgvO8R*x`7ormJBdIu53qgoN zX1EzS-!h^v=fzffqYt{&7B2=ZENBhI@r|iRExYuHRG|mrB(o~)bRm{c>h>XoJ-Iy` z|A{1L7(}GD?EWLz$_oA!_#&Klvl`Z~j7-+ypZ9P&t?(Ro#zu~<@p)bmvBJ>`OHA;!;zy{n~mXU{84)|otuK5LHz2-XfFFQKpwnUjt#MqJQIB{G#~i@Xav?xFE{ z#JP5(mll4uc-UF98@(rv`hp*2xzChlu3{32pu25%i|-7l#mOtr12|CO?(MYphfvA& zW^WvxA&aH#N0W6+0V+-S*RWFmTY^zS_Uo|DMN(u`jCREo`Bn*R3Q{27PW%t zWP`0&#MYnPGfyV#m+KJRjyMxhR&x8B3Cz+lyRi!7xIc#J=wZ)n)L&;X5!64E{3vXZ zEssBx0NJ`B-0FKSGfjb8q^6reM%v)yiUElz&d3%JFgi*U9#dQPcU{>+4^hZsgY?9Goo__T8rd1TZs^cxHGaYwx3D~F9LHOKK!Kj7J*kgN3F+0{c z&QkGP2(uLzTVeQF zGy`WxJ+DlB$ea{7M0j>ynnrjy9&M5@43mS$in!6t{&(5-(<9pr{r_`p4uKXaON zibdTXnBS{eX$wy{*!qG!VNdfiTT+%(v}ItFCW5xmU7HzN^YnxDzz)q;WN}Kv18U+m z$u3t#xK2UDQyqH6Ka-RE-qPo5#CnPNo*Y4jCTETK&<0-Xh`zg*=oOr{afFuot))${A2bd@JWELl z%s-w~1bGmoC+%pyGl?Rq@j%R8UlyliWU1oQb%EewAAH>tE@cxhv`^J|Z{GGptzl}@&`SV8*9qI2(4 z-4OB3CxW`M!C1;zu=F~y*BR+UjK24caxFP8(v@C@XL1n08|!JmrA%a(%5Jc|@_+}Q z>8;$2*Sfztz`8@wr33H!(;*YLRgf92Mo|C6OQV$K@cFZ?8EQ^_y%h_YTD$MxN&=gjV^f&M&+d$o!ZzE*c;~@cEyv8-FmQF~@C7x& zaHh&wfZ=5l*u}Wx z6wLF;ijc{XAUBdjyhSQ~=uM^Lff1ph4;BY95*hg$iQuV3K>jNdX-bC#UN$;y*Z5Q{ zv>4ru`ZG27LD_q5NZ)e2(I5!kl_V^_D}03U z#@M4Xhs`~jlczT(IYiKw`39!j3F|{8@aBM<)vmLh*CLDe&*!b4`M---X1z z*JY^U4THd|Vm_tzLiXT~SyWIB$Qet+385RyJXZ8_iDZ6$o>b60q^XFt&V%+(tMmH3dw83HGPmUG5I|YFtP=tB!&!}=hG>}gGH-hNr)X!Q3A1qPD!S*M z1aTNfMIIHtOFlN~0(YHbC{j@4Sj59D^=UvXKg z;j3%nmjIlyoIP%^@&bc}Tm3xQ@in5K;4bxAQ+fAjjIVTZz|$^lB;clRWPd$efx#hy zPMUdx575RvpWAn3nB4{W!Vp&2)uJ?_{Tq+ztKwdOfQxcDo!6EI7c8PYDLXv|FOPbr zDFwl;9GfXBv&KbSZg?d7tmK&9J*O-)Q^*$K-F2YKGSzTPGyQ-fY9{ogzjGp_(&&`x z+3pmxDo8OU&r0<^}GM~4z?I$TLy=HKNbw-op4W_xbh2rhKXS0gnD2|kzD>G=7yf_l= ze@IL~OdIBdIFU|$wi(q|k7FD3Y1lwWDs^lVpI&+(tO4uLInGBYSMi*FG!9f;k9oxoXT#E%58XTN83VMd7@R$MM=ymwi-%{vnWYdNc}Equ=ZFqU~b z#ZU>$Bg75=Xf;2cZ}nnTlFaQ-rYtz!{XXvLyttK23g4nvVWWk8&EUXoOo1k}4nTM@ z5p;@IXJ2MYWWLNtWwH>cnw}xBu~l9+eK6qjB;=j-Ce4nfg-GouK=_6;djfQRkGS86 z6BVgHg5F+3Liz04!okV3-h}^9 zU(_{%$nXcRWf<2nIgMw-5!ckk)b(Y#(Z#dGQZ4G8o_k?uarzIF=P4p8Vr}MwJ~n&N z_}2V!QZ$yhfR9LYD!f;YKo~3PKwt5N>SUI|{?z7yh7#scN*(#bXJZ*p5=^jBk-^?Q zkw*h(&yIw#m(@lyQeG{q#T~w#kp9@6TH=Qv_SM62t6eJ5@~1Y_=hD}uY*a#iZ#8CW zOqMfzxjbDZC!~W<#9NksE=@MMi+X)~Lz`fSy_9|1*~jm)^{jMf=Bbirh;;ZK}!djo?fSr%OL$w(z-Ji`A2ge976qA9+)9xMsW_FQt8P+J$eu zoX{v}e)K1#_4Z2b=D>g`ZaJ%GV9G&`fF?snys6IN3>x)jp6FdNdx`*SkOyrk8CG%P zm+MAE1gpfS{Z4E|c9Z|$)`3^vB4&W3?*q3@Px=}@!jqA;aC#8CU$i)ldqEVbsvo6Fu2FXN^VPSt|@OH{*Jtkp@V6U z8bm3TyiD;GME&oHgHdHGpI3?*Mu zU28~H!o&DF#`cCywUKfe=#kZ^4i>d=@Ts1Z>h=?S z_8k}l}43RBUjkvB6P_`o_A$O@b6rjFr_nCA_R zJI<7yyxKy5Z!As0ogjeHqlrs!Aq>*AaS8MWX4>1HJZ(M)-HzpKS}n zcuKpoam*BOCRxLG1Rd0H;1tf_#{yv-bUIh{ew1rC8?mmnyS>Gj?^2*Jm};Bs`)A=o zln-H??g}h{41`sYdgNABra0|dU*mYu53|%id09?2#>TU2yw(DTo0Q-nmcs)3u;w8dV9F%)>J*Q(D1io=;y4f3{GP~^t$aV|+-o60I zd#RJ%?PYqUdY|KicxLD~&F3j>F(50ReL8zCJqo1|h>BAn>r;ckJEO`+IPu6S^bKV;d!y!cHsm{cRXAANvqBe?T4PL*xAeV%clw4d(H&2a9{6c*d_% z6lGJN^Kh;AMS7<|mv4SQIcs)?cs6Tjw+5*kroWxZcX9rbSHC%>83FM+DP3LfJ{vob zpUSy$g8eMGU}!s2PxI3t%T#UpnV)IJgZVzd7gM>Z9O~Z~Jv5j#_jB9JFapxxR#wWy zT&n)u-B6TDuQ5c4YFD+KMK7=DacMOd@weVu$K6E~A^$V6A&C|AzSnhOg>!}NZ1bJ} z;sTU@`Zbuf#kBbOo(fXVY5ca`1x#;EsvJN}xQaT01uR+zF@`6Y)LlRX{kCqRBag~p z=nqcj&)(fe?Y6-z{qk);WjkEgJ!8f9Xn&-|HrIaIg*YwJps~OlUa-)DO<+D9k7fho z#4&bxF^HkQ!eoh7Vui6V4qSlW7~0YyxQ+Mf1YEQybY0=I6gtFEL1vUtrq3?U8$Zj{ zpLs?=9y^8HlQ;4yLSeT0DcM~>=0c{xgc6Of{8uz#CRR*9G({M|;}^{u37XUie516M zAq!8FkU8L8e}!7Z2!^uM-R9TPMy|>B|H+yX{;bMXkEaFNC@AApqE5jEd!(H$>29tL%;B{bysl>_78>t99Mf1j%Cc7h=ep zRGio@hXB|N8NGvNm$;CCuTa=;&kG!x#ECMYR#E@8(u&QWv%Q3}aleSe6FQ@9T*v7H zWeMs|5!|KT9@0hZPvY(XOCdy^TJGskax{5j)!y)!#?T4@AKZ>L^eIB^(dk3$F3?2X zUYz3+aotC6+e`=`Mq2AJ8?34M3NPl=l03*QTmH4mmvylxUIF^RP4c4#a&%$ixtHzX zQ8&4w-NO~=aoL>1U*JiW82Xe@Llu}}&;tK6b2?5QAdGxFTi{hVA&uwP54eKat#&=9VTX6i>(`4m|o#PJ`(gYh|0Q^tZL;1lLAluRY%Q+WO^DL~A! zfW33U`sNrf+voP?`ZCgD*W>zv-g7f5ppN|&MZX0tiZwp|gmn1wIUCXo8nU9o8(19K zGNrr$iwlMUL{MFeOJ~BlJld;lOsNKD=SiZN0S6c))s}dsz6bAGz7|9b2QAUQH!D)b z1_|aBV>%Q<8WbtC`MLV=QMH~BOZo%-Tvh`fv8MflgYQ+$)HF9KQq*Y1Ju!iah57_f zG+lhW`i1Mx&!>fpO5?$jC6s-K)K@i8smi?wfz#Bsk6>U4J-|9t#xiVTjvtae@U zq`TnauyESE^Gw~ADodE*vn^_VcYzlK+F2O6EIbp;?@QbC6zzc{);ld(EAO3AFT!$k zls$KQv$wl~jWt2f)A?O#BZ#1#w~p?o?U5E#_&rCjzbUl!v689gI|W;#LTr_S#)~uB zocEwyjQ{uKRH_XeG6`PIL@j3H$PgLmO^_YmvYPL?+F%v9 zBFy)G@)G{UMRA6!HAh0)BaJ?_;Ae(YkX8^f^4^rWH7Uxy3_!Dy5=*bazYWrO01jy# z@E5N7L-Al9mlyO(hLbDbjkuozWM{7@`>~apTNWclGs!_qQ8844hxmPX2|W?v>g*LB8tR>O>INOD z$g}$0C@O#VQRbTocdznvL?KmEufY2NTTJc8BCbRTaXso|k6U&u(i;7IifA+!1)4es zS|)PwCJvyJGU1IOzP?vWw+i!Yd67;>y#z#16>*3&K|&Y_4_3B}yZFF|p)i8cmwbnC z$!{;)`WFl`+VCf2&(UXPd&E)sy^b&XA zN}d8rfJST?y$R+&lv|eo$m_|kX|hm6Va=79t@TsN%;ymvM*ETcfr!1EPg!eiQtni)RMk!aUPNf5T`UxiK7on(9{E-cOb2I6 zjix{8!1)F&I3Lc6`K=*LtikaUD!&RC*3awIHo!!YA%dbF5{QRP6hl;h0w*Po`Fxxa zh0p;X74F4SYj7y}riKVg7aX31^a=rJ@{9m56|PLipna1-4(cQIKMo3kCYBrPBc}=r zyVDgJhJkQK{z8LNg#i{tBErjrqJY16IxEB%m+5&mnTi}bk#Nyji|9}5sBIsLu!}slFXh?xfCf~_ytXUl#k!_p@}aefLAP8T@A?D#1L=c=&k5~Czb&dPZNlQGNF(Y zAXoi!#ezz-h6%Z$JhZb}jaN~}N%)^VvVusqPQ0iM7Nd5}&rfCSiq@*0l_a}vVX7U^_2Qh?#F5t@4cyLB%v!n<;0 zV&9ASdUZxUQoC8+-DMBw7ZCV9JdC%{>c36~u0HVDavI!1H8YB!Uc;e=}2mh570qwcJ~oj)L*m-HXDlE z>`xQjf8VSaLYxre<_R`fiq=1tB8Obcf!Q9zXC|-|`NMme3Hbm{{}opf7$HC2MDTxm zgmzi6s~_1)nIemY*nUV9D%-{jPOTcpB1Rnbgh>q1?t-^3VD((Eh5;#iHH`gwYk)yi z{{1>2f<~crma+XRP{KpX^{?z~kaC@ISYu%@v`22{^bXiem5JQn%>*MiqijO{@0&5q z_5i%h*x^i{#pUJuxAqIJ9V2sdl%=Jm4Ih<2N(Y}lo z>_B*u6V2Y9y3T+9ZDS*!Ps@b>6+!e6zq!0cXC&ky;$dA|mlWC`ou_4YX3-qT0~$4% zmjc`xQF<|GxC2&FKiuA@HV`^J=`MyEAVyM#pDXZ3ONYFqsZo2z5KD|*sAo*qQpNo; zE?^K-Tek^77v~fmQ?c#>lJC8W@_-mJX6j-ii%LSX*o(|2VYAV=y+Gbaw*(k;FpLrS z@e?Hv?@Pk>(wG=abdJC399&&T79RvzI^Juu#+g=3xQhY)E*ijAbnjsH5_{M03eBBb zWLtfQg{LtGT=2nax(h^Xh&Tp}8w)D#m~-32k*-a~pg zXcU|x;K^r5k_QA9My9jO-8*gw1Op-p;PMNo8I)ofS4vwg8-X+Tg z*9RA0jPFzy#&@Yf?Prg!FrcFT8q+d4NcZBHp8lQ@MuA1T_SV!}3hgT+63DIG)6g>a zDSC6tAK)0JMA_Dto*jY7uY(L%yCUQA0VK1G2>_5^<%;>XWj!2wupVmjt_wP=TA+a8 zsAq63rAJ&H%FI5t$AV50VNj=mN)xkPToAUM#qQ$(-hNycl&l>da2Bk)3;L4=CnlX$ z&b&wailmkPUI283BGJq@*3v*RR-rn@e5f%*$aLAv)~XNK^PiVu5fErb z?^{U$2@U-65I~PpCoZk7BZyLZ8k0!*m<}Yi2-r25P_Q35B%US0gM&fJh#o*cOH>sf z-|Loz=EKy!de^!KfQGqLApepkX0y094ANKpF!qlQ^8&Qq^4!6dm6&wRM{I4-fByFo*pyh`^4D-ORhr)CF6z z^Sl@qS&(WbTnZZs2R;FYmJQVY;$~w>*E@RHu|%H6_CoTp(!B5eHq9q$hNma=%KAKg zjZ*QuhkGKEc<3@rD1!LCnJg&rJrP!+aRRwH1m)j6mX>yLz&!d=QA)?#y0!yRfV_Jp z`Q-Rj8v$Gm(;>jtgfS#NuYVj~9$M_4kfxbT2Z%)^mdd6}4o-xm-`=ha`A-F) zg_h$5#@}WRGqm~w?p_fHe)Y(^#8G1;X+o`qOrsvo#N`jvPlpHObw%=(iGo1tQ@YhV zBaocE!xf4VdS_XU9gnb_8AT=xqzosWgB@*vV`>IA4iD}*)q)Bg@j{2zMNwZS^vG12 zBGd<#(E<`4*Tm+3l=;jNK)2>%}B)U|??0l(>!Q4245jcQ=)5!@rO5MZ8)J{Q8 zn*tU7em2%1Qf09hrL^-h)8uZHQazSzTLu~hEifRXVdCxzaK9c&17JNbL}nv0OR+Wk zy(p84FSN*=0B4K`MP`y>YdVlDm7)t9M?eleA6R1GLyBpTOTpdoQ=H%Z=Rl`s(szMY zx=4b7V$vT?s{@8~{W}d&VLM!nJvy>4)~GS%0%O&8d7cSGK**euvE)H=8kL#<8z!); z>Fe~SIZS?W0Q7+u)Wf^s>-6b2;n6L>HA&7-%7D*62BY6?ro(8(A(g;?ug+i z#K40?((_Q&9%UA_VdC1N0`82PD6_pgY79Q`v*njuIVj+2drk*1lFU%9N0zh28z-PY zZ+_W6)8YH8KjaoxK6F5TcE!lklo0LRZ3226Ah}h~@c%yp+q7^#J6MqMIEADCTJJ?k z;diuM!x1$w00}o77k$#nwHd>^N^~bwT8*SSBUX(m1Yf)jVo|MQ@H!@oARRFYvKk%Ko%mU6AoFL| z??JI9w7nzrl*6SrsCS4~wmGFNWBFj!U%5q3YU!%a`#QO02{u(91^88=;o-OIq*m;&EHIW4rpYX^c-)%?C)0t{FQ* z;n&4l8{GCUf?O3!9Bvk0L|IL-jjaV2Yf5C+A1drjGwlpfO~&SRgc^M>)W<6smSTh(#3ZeDywHFwr2)Ok9i-q0jFpQv2gl{k2dxgfDjz-3N8_xy+|L_ zwwG0{n^8*O_?Dpb-33O6fIELe&!S5Cg-4|NHx;Mqd-*Sth;}qfo4*MeBmD9&9wXm8 zUDE}fhxx~fwV?anbCs4e`Q)k3>>lPnWMhS;3%e6?n@4p`^E9q{h&O=>3K7%Z{i->V z7m>-;BiT|iY2l24%x{zs@ZLCjkauGV!67HX_3pRFr51IQLBO&1FZWsvf91a7MR+?P zjBjr=S+qH#DQ0G7Agle*;9_g}Ka&I(*zmBZtBc=w_VYf{Me2x-uU;EJg^#vtA^x{> zjM=+3XP%SrU<_HUuiAm%tVXjv57#tn`^E;BK0mKs+FA=?SIei(mkZ$cHryc_pq`|z zx_ur&^oYyQ?zZJ@Yc-gP+d}3I6=fvk#o!60O3jN7jO`iKCT-8n$HrYaUPTOV##qAE zRw%MQoqWYl_2Be4e|4qvO&g;sJw5%8+X1{|Tgx(Too_(U?-iC&4GZx2FFu-JpuXyi z?Crb{HSmJ}#>T5!!U7pss}0ge|EMfFX#88sFm=0hUUnB8M8X58NcFM%Pt_aTdAu%c z>o~Jk;Iv|7+lt`9ccvSaymRG0k_k89wU1iFavGZ&%+@)gZ;svu1zyY0>(-Ln;-*ph1`+A*l#mojm2Qx3 zq$Q=K;mn15|IYcJc+PV^pY!s0v$uPD&s=NPteUmHYX?Wihee%90vPtQ9&gpN?yonhD#`HAQ3tio1J^AxR(XEAB=WbnV`I2&AsBc4WTmQETNsv*ZXbfr?tcK{yCO2`=S&d0ie*A0+?zw7l>_f z@XbyHaM+f20J;A8wO)#FE%h`B$JfseSDglR*mWwYTxFb7LN>RrPB(GAn!F_6NnJl~ z+!2j>@KQtR*9_i^<87Jm=3VMvWS(f*jox2;9gNGmW7y!g{Od=y$Fyq&xB43d$-Fgs=^PW-FSnK8FeLcHef|h@=0QHp3_C| zmxE46%cBkd0EOsG^?9HDd7NUsh)WyopOtHJ%WT>deRuB0IpfKMi_{( zD5zWN_xkEx29-q2qdJ#hp>6!JnOGkkkbwrN|2%v(I%8=ju}=W~x!MW!HR;}gFmCtL z;LyK77l(tFb=hu4p`)twHt6t>4oQdV0}{&^x`qux9w%WGv-9)RI%USvyG`3X-W$)B zyEN6(z84sma9NMqA~|Y#_s?`qG##N+iNgsOrwYEkZ4cDD9xJgS6*NPCXeY$dady5t zobbqwU%O=HUE{f{>1h}|fzO8V1CugW8-gqKoL1q3xh&fv=Eo0=JuS1BqDA8zp`eHD zYe&bON=wzRs}a76)9gB&Z?rjuGhY{m={CH6IM?jUzE!E_bcytZp#*E6*9e}Iwv?-Rc-72_2lRBa(L6F zaF)`qFpWbNLw<6Nd=*5UOH!8N%2bHGzwpjBt1hr?ejFmG&miu_qb(_m%j+>Mpm(hA$w$lHtqwBTU z>Hd5d>=C9`fuD8BbQVAxlGb?Kbt2|^1A3#)wwD#GCi2T{uNS5S4$IVuK9xzHyw5oI`bxkMD#_Uks$@fAs zS?V;GQbF@hO2gD6hf=~?+0ywC0^yqHvXS=dLT32;@$7eeO~gYqg<_McC4Lq?y(Jy1 zPXFLl7w$!dLO58*_~QNW_vm={{B!?8vu~9-LI`btIwu9FWuW_H zq-n3PvH9(4ySQ0(zPlM`#TKX`e#OJk)oQ?naleVDlTqv(ZK7f}nU%&4_tIX>008ke=`O1Z`Jyp1;rw2XTxB~rZdy?94S z&g1Y+3-o`OE&pa^(JCOVu{C18)@yVME~b|b<6YTbJQ=cCpR25HxGGd{(cZ6o>$@Y_ zO<4G9>m*aH4Ur~#oBntC#UD!Yu3^d`g8SzzW|EyurU|7&6-aDp%<^5dgk<>`R zU5Piz12%!}m9w*?KZ^73{S{LN#buNzgDmUO755|SPrZ7y(r;G#1Q+{L=d5NU=GoSz%9V zhQt<^JLsM$i|YGgV&D zn%Ec#B<%t3u2<$WUnPh=4MU-Fn5UC1*4@_%aF~Kb=ef|BVR-4T@OS-~=mb7&W~3w5{h=D7oqw^>hV{I8BT_kXpUH+S*@E&_OogfX=AK zHlgWSb#nD+n0Z~lj={QW)35QPc82WqpMuFu`io1iArUt&c!^HQJLGxlDCZoDx%mec z13nF&r&9ecZGk-E(`}8cfeulbm=~%XKlwRP@fL;$8DnYcCTw~B2Q_}XK9^P zneKRGip4h=iYR(hbs!zAUT4dku!`nuLBpmC!KoStKRv$qmf;r4`c;-JljS)TZ^|aa?FSe%L?mR8xkoz7Wj#?Z^ z-3&nN`ry#Vx-n{isCPS;Cl7ERsykw0{8b6;OY?QXRW}a4iZz8kf`e5Y61ABXV$6p$sX%+)P z`6NkTIz-gdzDs&xwvI^l@~oNeRX>*FGxOD6O z7*xgXM^fW_T_5uO*49S$yd9Mx>*43x`WK+o@U(Bcoo;(=cOulh87<^)9-*(e_$m82 zRxr-eV9?RY?=yjR+ZnntK}^oIw&;4dSC;d}4PJ-n*B3`57nhfxHWa_=<8Xkx3`lWG zjbzI23q)AXHhN(IBy_9}16tEVwye4!lJmtwxh7tVsAfa>bB*7>#rQ%9qFr|$8b>gM z<8jjm`q8MUP)t^kIY(lGdlp3MeNEC(61`7I(`K84U(VKKrmGuH52fPW_+REO{m5@c}vr!$v2SdH*d` ziTQp#zEocwYjmzIf!9u6vBAGu6|xq{7)C3WAmm7#B^;Q###-^5%n4F4A^5iw{)^a_ z6MplQo)i8GRr28t9HrAA4lrKr`HnN<8NJ+-%I80<3Sv7qYi%ca@9HUO-e5ga zQ8~DbQB@{5^JYrDZ=Gf}n|{mM_xYJB{El29gyfVjkMo;OdKqmi!owvwNbkBO*o~CW z9=FE!5ySaZ%_YvoUI%8eLA_i#xWqwBDV>KHbeQcI6@q(~pLP9!)p~4aH86Fn%{9Oo zoz-6N4k;QlJ}uFwJRJ%|teJ};L_I(MybN~ZjSOaa8~hqbQ|REwwf8j4+~9tQ3JbPO zr(FGtxinD&$CGtrrKAS~Y>R=7s>UHOCU8paEZd%)nHdCkXIuCb+I{1 zg+voq+pPIbyuy&oa$2!)KX+x3-ZxX9YdrYC{im&3{aYKN#!hf)W0b1G*mN(;>y)uT zm#4CUNs*A6+SlE~$3NL%3*FfmU47y3mv6bJ8&<&RKq6n{P<97b_WS)61odJo8C%(B zOs{<@!|Rv_nFtC`;J`GB@t7aC8e(JAcSHz>SH4K^w8_F=IdxV&#S)GhrvKzlD^{j$ zhqf_tibgL>@qM)~A_6AX>uY-{1n2SmW9xOgFJnx?7^Zv2VXlG4H$?uAX?GXa96~o)G1r{C~2RqU|}(^R#wr_H>TMZmNO7%gh`UT)_d=11Lt#z+ET(*-?m!;T&m1HIx&HihEK;->WM4G%739% zI4J7IB5v1L8@c(D&&hozi47Y_AAG|4k)sfKZ`(5qT*$Jo&)~SR@a=6qt_}<3gd10H zz^vJ*@6`pX^*Ag2fhNO~I21}@LI2?MJFLbb7bH>>xXh%|crfnPmyuO3`EDmZFV^Ma z&2;UX0=+*BB3|yQVKj>|?R^G1Lz&N?+s?V;c74cgook$a;P8!qYr<8FX;sY2y;+I) zYI|C6q|~Z4(Z|m^uE8zkyaQQ;YI*qen&J#f@ z1j%^Z7lu}WCumsxvCMp}E{ah^-0hL#Z)WQqx>E3Yo@j`G}90kv*Yi9Yvx&7 z*>cmg#Xhp{ID)asm`vxJ;#iDDZZWBfF-5<))BI4;Q21$)mTZ-6qRwgo?ibG1mvOjfkJ?6Gm%BRlRuE-qW#RZ~=!pm4 zaWldMcF)VKu?GJ`63#~LsKuKxk|g> z6dTc-1a&+~P;+HeQ3@eB2YsS)?&&fN#vwLo0hczVv(o(ee$jHlf_AN2VvV1`#ToD; z^;dg~$<5+PS4p>Iik=3iN8&tEO=D9v3^9aqNi9EoxfIJy2Y0KNF%YiL+q-@4GCyj4 zs`!%E;@%#L@A>yrUA~ueW3vr*y`preAw3=MSn;`~7Z11?5k1R`_3@mNJ!UN{FR>g2 z5^(jvr5oW=MC=O5(rJjdn`3mUzj%eJJ>!)WsoXD)hVwe%L@Xxi#@rUQ`-)Ym6X3Q) zE;fDu7Zqpx5Jj#M8*)bH79wxRwV~PYK$zd`4jU7$x|gZl->&NRe59peYS_&!85Uk1 zm|erU(c{?wmLRa)@M+vltt}H@85GDtrSK%uZ?p}uM-a3)cR8)w@EgZ=wub6?`#tGA zrAcs#;5*!bX)%>I!8i3oqDdYHE1_35eQGo~juUcT71GF&XN>kYY4*_rU9hNuxDP8V zF46e%xMLRs%gP+kgK@|o;l*=b(-Sxjq?9XN@z(jgRb(-$qnrA?NekZN)P+$OYUBof z{w&!3F42Fcc_sc5v00iX0uNa4=TWP)^ah=Rl)H0{G9Ur_doP=LW(J@Bd63fn88a=g zPZS^}^L07&U}Z*zD-s|JXv+=oTSEy*hVwZ=quHJd&|Q=2Q2G;b6@65w82 z@>f?E1cnV0chCRK#$9t6FN?z$-#IR^b>_^FaETy5wS8p;p-$2i1{LPu#$kY3c}L=^lLwl_(Yyu}SMWq-l@62mbePPDVR+!y z27kRq&#dvSFT8s-?p+rr{dU+Ul~i&jZ%&lhbmpnX1GzezIOZu zE@9f+niEaD5N;Tt0~IC_Po^>rYjJxN#qkp6=x7`pHJnh*T&YO1O7rhKz{4fvgS$-Z zv4%>uQtNo}{_GFYtJhi!gxI7?lTBrSaOrOCBL#GEKF_1~DBlim1uu7nFL3{E`xSJr z4Kus1b4bG-S1CY(X8$Fs7<|ZS{nbSZyG{sN6rK0&RyXJ9d|{EfhNy9wP)(lXwf^?v z$8zR7Vc?FrCysRo%d8=rNk|5({om$dd&<{);qJay;owfGtnKykeQAVzg=nq4-C<(0 zT9|S~#S2N0?;Lu)Kr{2@6>@fxK)Y)56Hbn{KV`yfeSVcB zx)2?X+%8+}VAZw6q+lCXvdDd*kW_SOLh2jr5#*>|N?bi`&qv0QgpNx9I3XxjJ?=KB zVul+GkjOyW)FeQ`P{zFD;^LGbt=Szj>NA~ecmm!-tg`X(96ABSRcy1Ho_qqmJZ6qrQVtIMpV5tiU_TzALepTm5UUU zF#Hw6_~e;p68SXhF1Vmd+_3RXW3~ME(UD+Uk^7aGT8N``TG;vVJgxI;tA;!eVU&$* zFQ5Gk>Avcsc~|9&G^j3-C5O`~gR756C%-YW^y;USP9o}z6Jw6M+4?kYMxJsq@uHZT z4T)wemRYnA#)C>7x0$Oo>Q+Z^4Z?(`OiYwA`O;!k3+X5G#NH=# z_!c`xUr<6d^FD<3^a$97kI~C^6L0~1-ldl5^&dLi#FMWYKpNMeto+NtYH)E9?i#OH z#Fl1jl?Uiae`iFN{>akHdQ&v(<-B3dO-Bph{CA}9#M~SrY5}u} zbuj+d96m(cK1b1B~Ts&9_-Re4P$yTg%n#zNz_7O7ieRgBZc zzQa7Qh2(JLi*}BN2vm3G0>NF8pi>`Sdw@|^x~`K_17$H)Wl_0)dK1%1NMH)m;lbZ9 zD6AlSo)AT08!=oG7lS*5dTf$ft5tqR>`e}9t{&oE;iA0FMSS`>`SuQI+a>#TC5{t} zD}q7-00j*`XIfNcIB5kZh?aP@>Z5k35eQ?R3ViY){>x(E5y$fZ22)Xh@F?ksgBrfx`%!`4 z4rc4z>H{5_H}Dw%middr0gZ5=qxJ}(g4U-RMaYUBj;&2WBd)*#LsG$WFonyOp-RLJ z_*Vizrie0%3ZI0LSnmhXPkrypr1wa5*2Y1$zc;#srJofJpimYKRocmWg<~Rlf}UjQ z8-3X+ku?DGG#b}z#d!nkPX<5o^!(brBCtfeL!4i9ZJ=KSt171c!m9i~@8L z_GrIAXdbhSt$s`JlL_o7G|4TDiD-c6@%*ti0m{!%cM8-{%O^wZ zd^6l21vDKH%XQyVkU$2jBKrF5=~D=x!{(+tLkiHP;BSaPqfF61UwR-`{XorC4my84 zVg~S=wR@mH`F}-S0kUk4H#`7(>-DxV)7IE$Pk{%*pndigzVZkUfGIq3<;s_ZCeWq< z+lr|+I>Ok7B>->1v-`O^h}YN;`xp#J+So(mREMx{yvF(;`%biMsD_gp0WNS=yB`<0(5P+dN)0XJi!_Au0iBM_-tX-00f5-*pr)@ud;Z38 zR1>E;)O+pZYRwF=l-pEU-ZyOF0&E#s!9f@R*j8Iq#bX&TS}On%tMK(ZG!Izx3qJ4@ z+n_0~TRdXbSg!Z1L8o*K!XE&@z&45ga)uy+_=8m7V^ar-bcioE!#P#_{%#jTh96iJ zrfGRm5rj#P73Q$VUef%#kpRy6&zAtSn3M&=bJ8HEf+m_w=R07R{#SGyK<`z1x`+a- zUY#p9f$QdUJ0ghH6@>d^dIrKPSSq}VMxbdstUE&M2mGi3nMs7baXy0HnqhXGjcUiy8@F z%f^0iTM5|XMfu50Ixw2;ds{bysTueawCGAQ@RNs+?c)7O)2Gmc?+Ad4lDgX@Q2qcm zkE9*K9zwcg6)6!z!Y$I&?C~>GwCAN`4p0J8R;6Nd0ce)HoWe~0^0*7D5AfgX%EUqn z^MjL^FKn-{O|61H8Yraz203|@Um*BXHLkqtO+NRY6ixT0PTG9BxU<{)UytVaHFI9e zEHXGg71?t}y7zHHEXk6}{{zx~@_1mN{`~m~*|?|(=LmT{EwUYMCzUuAK;>WH5o|J#%OU@Hg)EAadw;A;<=l~ z$ow0pA|wt;RXX3(@;qyOU6Fhj4~L_aV;a>DMZdMUO1p(ff)WYiso4p$xOfWuoGShD z;;9)<1P!tcLCEG-z#DsvpZJ_f)xpoM8B85f#Ke369Nn?FX#M?OA=}KtS6KkCG+lGv zV50#VhNhB%aid{8DC$kVpeePD&#l4dQvG?}@JtAL$Yxe6=CwHQzgU2BucLT+r2e9L zCH;c;e|uJSRPjK=>SnesD0JQYzTAr#e~=9f&jXmD_qN2vi2>=Er8JHdNPn$zX0 z@|nOocDqRapX4+iV9x_icXo{3_j+$FCWs+wFoeK8Qr%jmuap4n+x3>^YqH1d8Q8IKgc++SaPCyQN4)Aa>)`Ngm*{=VvSNO-fV4DmGTFtdp;}~cS z9<96c60}D`msUvN?nx!k{qOChO13s8$I?Ep|Io|?9p^@L30`T zQ(h`Y$P8N|lDX#WQ*|?cJRNz_qy2WQU2dgCyS34lpMA`@eER%3PNAXvTwJ)JxBw#W z0NMWGY+#{&30~>~#~|LctW*U&T_JI!Z1PRA*c%npTW%A8K7++lGZSjx6W?gTC%>{N zM4eFEp2Q5c8MZQH$wa(aA4vK?&uxU>HibM?-XOC-c&|~sGR5Zt@mn; zmO;-afev^b9z~-BnSAB+;2>stb+`^%Q%lp=N(1*2W{e2;=KWaIY2N)rm{fg{Mp!zT zfHn+$=mHAzv}ayLcg25=D*{maQOSxWxM#zO&b}6t4Yu{0vgk4pg#7-iV%E8 z%&R8{nd7WZ#y8r+2D++>x+(cV1C~S}DrmRL)U?6}VNEHm8@H(@XJ z^uAI^NK1S`522Ay+MW5H=y2PePjuJi_+!ykv$M#Kl-p6syxs0EpW~;3DCq7C5(nS2 z&3TEyt7%_lfp@j*#pD4G<0_+!C?r=%a)8$)QIR+7@hf^MpjT>9y>K(&KOYk(-U3eq zo0dc21#|Eq7>bF!kwf&71|1bw{ymb5pk$Y25qNQ*pF zr1s*A1cT=`TQSw#2$DURp!Xzc77~#9usZN%YoW0=z)#f0@hpNL)uI(dJ0hG#06BGl zoL?`H4xopfabL;!tp>6u(y2hpRR_zZRHiVa_mi+i8R?oRfPwe(XiQNH>Qm5fh)1ub z1>A`c!DECzRSZ_!pZD%7o&*gO@`kz6AjNGk+j@3j9~J@*LWPz=Rd)+nA{)dEh`nr+)CA}gfn8wH(rHx-`dBMBUkNqCK zl%om(ueN@m2YRAXng~744eobF0po-2QL<4!di~yYdn%Dh*7!h*+qlkk+xm)iV&*q{ zV=pA0Atrux@mU*xZDg*0^h-#a0;v}(;TqnE85HYe~d<1(o7%vXS`#Nw3_hTye*6*_i5S6O%&JF-jN1wdUQkz9VH8}bqWjehJ`w<((z z0^TSESQ`7EFvQpC^%(L;{9+9Bp8H?0rZ)eJ-QYLyM};Vj>kLs+7a}{yhgIf~R>q zb?J{SO+SOFb}mpu-#sFe4z8;lL**=9({ec;EV9gN!?vLQY9)pYB29tdK_4Q0%+}M8kh&T5a8KQ(=uCjw@~s~? z%7~E(ubw;Ah>`d~cX0(h@L}&fN91x~}u1yTjZZ)wr&_!F7ChcDb^8WKMzBi=rgggsHi0cgZD0skya`m=`l(sWD&niMjlk zpW)^E3><*{T3Nw*Vg1!=T;dIJxGdGy-u64k?9mTK&HEj%HyRp*6BA#IJ;W-Wqmi1e zhCNPflb!ABjcmkI13r9y4cX<#?gg)~AT-x1r`5>FY=(wyFy!A%lO+msn^#u3^q(8( zwi3koi9J+|58ao9o?wnEA6xi0wz4QaJ7eovDaX*ps-#t zcj6u-abF+$YlxS0^%?Ca+nM31>Y`K^g^ZNqk6DGkMM5z{=eSZ1s3Q-`Nqr7h+l!_C zg;65)HMU75;@`T5E9lv{^noYaT$je3la1Jj23&LBV8-MC-O~c zKmh~?sY(V58HM%!g%h~`F+hi!Rqy>53j*V7EEtyEYvnP>Vt(SHOZp|Z;No&py{WTt zbVBbV1Ki~+RAg7%FxR);3XsS_8JU%bSBeR_2yU{{AORAP{Q1$WjB_y7y6p{fYCo_p zDt!PUhtG@Tx=lBqL?aOpqqwJ3x)5%d*DS-hPzu2g`J_{h7&RaCY(dhK4o07(7*Ub0 zz+Ds5K0?t@(ic-lXJ^vh&=Oo0MW7GIr;@3M!p*(?%3%4)aIE_RE`kMw)Bk7~h(~I| z11NAvSZUw#QlQX62XK_Ew#RC2oGSM+VL*OgGd+EYrkJXPh)Bf^(E`?-)nV*T{*I+6 z8Y-luH1G3}`Vjm-4JRWIggU%3DGK0bs=-}bO}EQ_?n;Cl2qz+)Yu-tv_Qr{H$k0>+ zUD~aDd$T-BFnXa9M04qtJuAYLL^QN1alad_;%~?SP*|5(wsg1UOp=a%mWL5jrv{05&#PD$gE) zednSAk*i4mG_-NiG;&Yc zg(NU}TDUqec4$$MlSVf;p;=mOW2RP#eL?Ll5qDK3F?La>xIyLqsp{pt`a9vH6uK9J zi@$g}1=uI+_!rX5nV2 zguU{<-@0wnyYvvlE<;3fHnRu1LSOI`Tn2tUiBF3c zOLd4VG%1PMUt!Nlac1hn618vf?&+@D^@%oOdwOq6@qq3D5jmZoL;yz2iE3}PNP@Gbhi_J{r3jk9*yzmB_dwU)LLrT{o$yz}7@w&hODyDn6SsDC;q~C#;x8d7No` z@%(Z3<-G6tU3i^?4(rpRm=y0h3tlhN`DZ%DBG={)@V&_ni{Wp)N{Y+u`{BMI(2T2VqG$c`;ptzX;2m!=eOv%|ar?%*{#GzSlPrEP@9% zF!T1f(W?rM6hXOuYsXOjM|;?X^XsTxg|&#ELnhT}o-@c!qsqsftZNrff<)#twM%um zywfQS3gC`Qj4!%(JjXsf{QUX@8Fdr=t#j9?@%({!lQ`~JgbGZFiu8fvpF+=!*Uj73 zu+uGKC)r9w&Z+pHcy$L~wmVc_0af*f#i={ZKaX90RQ>dE+O{)V{zM-cC1zwFHOj6- zIN6YJF@@m$q)Gx`{xYuIB1O)aF90*qpM8~|Xmc6FoIrZ}c$0RRshMf%1xJj7*m>E^ z(5lFnoj;{NmfvXeF7FvvTpd^cSlat(nFeT!la5+6n6o$Ezv;1uCj;!l5-Ule)ju&Pu zUx=qN%*>fI3xzj4{Z-gkb;DB%lFnLqx!L8eW%ajK9l7rObT6x>-_2Ruls{HiRV>lk zKhodsa$d?OXR@P6F>0=9I5=X1TbxK7^drUu=NL)SoobcNUKeT=>i5~Y?Y)%iEJ;1@ z)=BrFGwHu-YiL%5r3$tBbhVhJ3A$?|6>q&f8>c~-_??dk8zs& zr{__9Sav!5kAA)$E*^}W;}AP;ea!oN#cjEw@_uJ+WRsw8)2k~(*zU>OUfnaLMPs+m zu^87z=GyM-Y*DKnW6=tg@!bXAF3m4`TFjT1^(>pUmc%dj>+|4y7XhlvgY=OPr)z|E zB)Cwb@p4Nc!E)dWD)vL;FcY1!m~pYR z7gUuweF#nRbneep_H6aqAEU%a>Z+R+I<6^MD)XATN3FNGws(F#4XIra=C-_EdmD4^ z%i~<9RigcPYHqS+tJ!qU&FNAmpJ=jSFi1LI?CtXIwb0~N_FV#_r;g*dF8d7AC`1{> ziM`Fo3O-vVuLg>}Q(Q|qh8+|%E*B1It}MAR6MOK$i}p-B)}FvV`f$R1`}K#Bh6aqr zjacNUeZveV&Vs5`dH#2y@%Tq*oPf~80dM+OXc(E9%xD(MZ-i#=SKr8Ap&^D8k)7*s zNVZNtVtG3y-q)CK`t5R%)LNIJca(V$*h!$jgvpW*tJt-XjA24qry^7S76EVPp;2N zc8A%!lP7DEu`3b34wMh1sPE?PUJXWy-QVn`x9J|?_Kg~s!{HohkQcb}_P(7QOET3^ zQ2T_Dub(lWn$?JkjP4Hg5Y4t`_^Qm*>#Yz8^S^e#us$j^0Xy7J2KdD9CRvW&)iR5|G^lJCXkx!qU>uZn3BIgiC@>hWC*Z2DcNmtB`-GPhaJ;B#?h@TKBc`KmYZ6wP2P z{y_Mpg!Nbv{b$2TrG7hjHQcwo{5&=~v@$N?*oN@)_3-n#Sbfi3=2`pZsRqsL0w5S0 zipfN0X%WDUJyebeVou>=P8rflqONZ~q~=k-f3d_s_D72vUL_ZN(INYb4a3fO9jn8I zsJ&Kwp!3PD&(rl;s*PxCFK_0chuk~x-wH#=ChPRIj|@GVl#;3$d>LgQt=;cKydIgj z<8Oubk2%j46(*xZ76#@#uvvQPY~T6V!m}ZvH8KXzKz(tpIIij8<<1u2sg$9~+>x0;_HEAF4KDYzuzPT_{Zh;rMh1a9SW2nvojys!5<#XLQj ztlh&oKxgw#{IlAZY~Hil=d>e!?|o<`aZgoOSy!jzrTJHcL(hl6gWNw&M`4DX<&}F_ zw`;t*=n5-rjxIXn&@dmh$?|ja4m~#ksi4lo)%RedBUXFOwNSV4GE{aFuu4_8Vd9~v z$kaF1lzPBsHQ>gYHc7DbVI7l&rTfo4*r*p#5z0=aFm8^5bBx$~j>HI4ucyy9xNqbZ zw&4&S5c6|CyS?@bB(l+$Rv&ydOPLR-+w{UrL_&E9j@oaB)R}M2BItzR^v*da|2bpl zw(~Rgzp>wU{0z7{N>{Ep=X)rO2^8-YV1JNg{5lujRoj%mbdY!*mq${=_04L}7DRef zFW$Y8PyhvJ>{6N8yY_8y=qDh{jh&{vVg8D}Mj8o<&1V^T<9Iuqn zIOo$>16MHU8l<5EhTBwOJ#->!1Lw2w4w!*lri$g0v?V%;1G++xk`XLSd@%<__UBM? zdcZ7-Z-K7u0F_Ua8SO-)zpD!E9thtP!tUb9vqG``@_!ZEXTZ%2WToB(BVyi5lHK#i zg@{KUTBI$ghsppcU>6=m7geF;s^qn`tDs8*a}^OIxcOGotuhS-1xn%J%4_hMhIQ(S~`L zSjcL_-jwa3G)*oz9_>cIee?^Qms69$(MAyLr}c)t8fy)7I>QACtkpm}G7K0Mmxk1B z3n5e~sGy80jR?-z(NCd}a+U>WWTm1Xm@$tMiOGd&aX}m-s4-WJqcYp)frQaP+ZG?ayOT$kwDVwLb-d;r^c zC_Mc)k+29hv3)6}BwLNSl5xNhPaevgoo|72E;#W)X`mAhkS18$+Mi$~@K8w2g{h~PT%G)8(xC@bm)vZDXbITXdqau}G5n{HEZ zn)v>nIh$H$ai_n4v{Sd`Bm8qvEK5Kk{~Q9&!i3c?I5tW}ztG@=#AEc|#A7s+c>Drn z#zoi{WLpn9dOH#CS&g2|lZ*QV1O%vZqhgP(@)M&;vb&;YbpCMG5&0qbM>2skJp^%t|;tRPV_}Wct{K@ zV-^Vu%=c=kuQdT31p_K(M7{o7u>l3gX#3^ZN57!@i$4uuCynCdd{ABTB?T_>&FcA4 zSd&SHD7enE(T@_t=-z*NJ&SmuGPPXozZQNce_z##ts1}=}i!{=`*q?H|*He-i(&?$-m zz_VIfb5k&Y$YKCW8Bw1*LtlOja%Mq(TQ}?XlTZR)@!6Zz2VlH}23jpl$$B*r)WuPW zGF1M_E7DMa0YdM>-#{MNPhtcN$Rf7%f~NWjj4#s#s_7ZX$rK~GROr$@guI*-ped-k z%k(cz(oBFRUj5rQ^&d5mP=nz>I*|oPN}+{jYyhlqZ-m{4rbj~gg*n(Duk-Hm<_v@a z+#_j7XaxAyH^3!S=JZ0a8f{Q!(lcAj_&a}$E^w)#T@z4+O(KH62n7~>IW{|1n~86dx8MeBxxm5m1NTrL4AAzlqaj1Nu>Ybk~-j$koe6S z=zxoOpkaI)6xMJ+tWL!c#R{SkP2ipLf5?2gp$G${O`{f;`CK5T-UBRJQd=bj$KVR! z&sVnagi-$0<<{_V1?dC&JatV5rXMuZSMsFumL7;eoEw$^nQLVVEW)_IZ(j^ zEDZ&5SyaJmK0Ro9A82~PwGc2pr~>}@TLr8I(VUBTa4)nz2^um${D-I)mZ9Kd=E`8T z#Y4`fpr!{7?Z4!+(80nRfQorUD>B`zcHs-i+71x(9zxq{i}lJmVH^7ukdIGBpm^&U z6oLct!oV_|#i_3XI|sh|5&YhSucrZG&l3oN$bc8%kAyM?$S+a8>kr@;`G9;?f}X$G zlLi8iR5nsfyg?_#gNVU(<*gN{Scm~z;SHmu196c7h!jA~10~U++A2TyMHeFU-xP&O^x*)o&Vj&9vG>775>@d;wDcuxJzup4e{JSeiC=UVoR?e;=cac$TL7H>w zJ=!y{CFt|+Lw^a9m!ok6hwS$gcaROxPIB`~5^1>r`)<}olZaxN&cwHS z0|dDFUm;S!DBL(wB_IU<;#>ay9@s7K`!{y$AI6}fpbs{n`?WHmPq2Y4{hyitha3M0 z77&pCkqyYh|6etQZm`5a@E6Er`Up+WQ`lc0*R(2A+}OyQfB0a6aPFUI_{J2Pz6GX` zyg~vS9hH#u(d&Yl*Tc$A9h7bfH{c+L`}O~TgV2RIl5a}BkTn3|R2C8Q}T3{e`zU-D4#NSa-h_AzsT&0-(ffB&B@2(`Xf1mNC7kwmSz zb|L@B^`@Zw?mvGC%$*h(Aj{bEpBW^aK*^jz@VIxz%zxW=w!H`|I9lOv%m5|?Rujzs zUOMlFOVB_55A&lPj@yZP{vp&hLM3|sxBLv&^Oee<@kv~gsbzl>Bm8Bex&H>llvp$-4&hacmCzYZxi5cu3?rzX_)I}(;K57 znTzXw^M6yq`sW7?goCneCm0Uh883p|Vd5stxys?Uk2ZU%?Mm@4=e_>rA3!icAzmYo z+$TxXVWudTP3fzX$L5>bQ8R|*S=~ZGqaSsiR9lboQgtLDcgNs$b>S82s^@sYZNAgt z{%2b>k&}oMF{jGKm9d5QB?$jv`+IC48(p2wPA^W+cj-QA+nvjilX%VG4z9|K7SWA< zL}(Q)%l1k`vD1xs<=_ss%S^j&H^M~-Ly0TT7Q3z@%Q4?Z|KrqvV^;%yB%l^M)xtlP zT_%o;NUvF}$N$wy6}dgW>j({1X1N&2epU$bo=m&o^G8`rz!zCg}5$vcYMv(`e7 z_^w-wFM1U9Dx7kAQ~2CEE{4ATzD6|-`j_aZfM~sywS_1QtEFdM_)&Ms>4Z^lbueT^ zMoxJ;MBSem>0s!;0ce76(MbF693S)U*0P)o<;kq|aP!;%qO5LZpZ``D=I2S`7n6JB zR@V`sfD0i*Z^Yw2NK|ChLv{6owaP_0Bv4#i#|*D?`3G3^&ksZV#JGyuPDzNuY80}A z@*KZ)h$7pWhNoYJVkbS|5qJ0Y(ppnFR(yLV_eGEv07C!SdJeFoX7D%Y+;{!ceWw1@ zvWWR$tS@D6jO^K{+(7*x1JXb|;WHXoTIF7G8DTsydEB*ai+1}Xx7WaF(ija_|oZ|555cAaM8u z4i@-9Un?x=5X6Bz_|GR;G9n{>bo_7aU3oavU(|2kY-yXHh*DCK5h6=Q)=CJ4#Eh~f z%Vf>akV-_dZzCc5HX=(HMr55CWXrBFWBn0h$zbNa!_@Cxp67ku-~aFYHP7>%x%b?2 z&OP^@&-vW@)tK8zNSFf`g|tSyOqnOoE8?cA=U+sMD_|h3gl|KE$2oul*d*o$)@}=g z1d86d@A&jZRH#y}u^@P5uhb6tjh!mfK#qIZy{!D%J*=+Mld$$ricOrG?>O>d%)cBL z|3$jy3-3qU$7PRVqz!8yS#bUdJf}2(4?8)LUzB>3`w50Xh;FHY|CM`tgwBzd^JD8*tFp zPkHjcl5HC7ldEULpHY07Od6JSW`4kDn8D>9XmO8bTeBk=Uo7N3GqYzZ0t4AAzZ-;T z(H}P;ta(tt1hz${`J^Pk)6lQR7sN?yB5g|hSuNhgZ4rS6$T*?cFc2~8;O>na7X)Ue z8klhHb7n`B(Mba-s0oSBantT0m%vl6^tb;;HlJg_7QfJXOikd48>BWM^fchd-vfkQ zK~d1+y=%R)oM3!Z6Kpg#d^d zS#3^T40(lGy_pMdnEDe?Lp2k<6$c%Cerqxi04^YoJ?XIdCvYDLP1EsdK6bb3tE9I$@7&kq(EOn zRAZ$UrlF_)m@rat$tdY~3!bWtvYfoN;?;kt$9uNoc7vrZt_xvIktRYc5-t7`a zhB`(#94}88PVkPJyq(qXnC(ad2LyfwjjXuc6d`u(?(2L3e~Q}lD1f6j_0o3GKO5yZ z6t2^>-lA|4fnO`&GbwS%|4gA!t~0(SlWEkJXeUd;k#h9luDc};k#H5i>k0R2Ebzuf z7N6Tv^**X|@bngCOUbSMtQz@Ma02A<5{(M1DO+U)Ou?jbUKb0wLLGkU+!8r?JG-gG zyzXkzx!c5i@1YM@mXL!c26?6;h>C@ zvGPMq*|`8S{G_mU#9XlAxz&!GvvSQNP@LIOVJh5HE6X73kh`dWbjqKD987z3ivf>W zV&_IQ?vQu@e-Qqje8;+xC+O(4?9*CA$NC%b=b#nq#W+QK%oF->ARs(YGz1z|RQ#H7 z30`3bd8IwkP`$e@%+WrhqG} zhOyXr(PSbO}vsSgQ?8MH8Z&R>{$X-5;$9 z1pTy_+kDAG9-5rJj}fS1XY0(n<;AjTb&1r@xGpQvju7EL*ok)hrDS8QCbR{a~+Y~G)z#7APQu(IvTj$Xjcsh1`r zU@J?JhV#O)leCZIie-!~crnM?&mL>b2`GMgp)5Z!3R{-pIGIW5EpCJ{Vi|_L3KKF9 zBi5!O^ob8&H!|oY{YyV<-e+TUkR4Zd^`lGf#3y|VqFH+c$=OqGIFr^}lHY{M3!rZm zQz%U%$K)DpICz`PfoZYOp@fT?iu7qon&ST0*ELbL%@xmJrDs!ebG6|7H z+^YN(I|j0P(f*t{ec9V`d@7Id$xlF9J=$&h%-B1Zu)>+%&Kz8ZVRMEHS+Phbr_nk2*?t@TMasN?)Om826=@W6H@-vG9MAdcoAU} z=b*y!U-wHFrazJ=9R`R&qak9ZHW-Ekei7_X}1|0odDrV%`8@`t2n&ROv zm&Z@Q#O?@cIaJkB641nxIoJpN-D{?~1COJ%?R#S` zl(X}lp(4Bo_(XEKNP&gj-FsP4WfW%H0_bP`Q92X!h14-l=th1R*V_E`jf-DN9H$N zsc5{^wIg{(C-7KPTL-sAZUe{wy5#NJGM+GC2)DsNYCIOiIRwz5BR?PryQM}?_Gsyc z3l)6U{GN6mW5-B=3;o@{)2{mv|1wn4xA#TOa++4`0uEj`obRM->v;xWPFW_&KzeW) zG2!8i?u-^nvleQgd+p0XDoHUx-tY}>^gz$guNr!lCZ+MllSO&h9bc_{y%zG*JmXPm zEBA;8lQjtS3EmD^$;D(XPIUi7ZZ!NMmWS8GEjA}snj{5v5o&1I%`e_(qi^1to!2A+ z1};$j!2G>ktb~MX9lMPO;rwZSd$`oY0plp`cXnQ9q}->CKDI23y^}N_ZyN8lPB@=E zh^k+EbS4M*?1xi4e>V4^uuFaRu?FhH9tSCvwDOnUEvB&U@)Z%s;XWZmH5qz4@`ivw zC25ZAe#2v4S@n%q6Ld_#mxXhc_d&vzP1%-(CoKA2h$6*c+bX3Bc)~R$}ALTlVu){#XAM?8Rxw#ocLNpRD;PyRc7c4 zTue+TBVif?D?DQ3NR{9Ubrricr8e}XN1 zoFKS7=&VB(k-O{=1T%~q|Ao)JE4*Fajp!|E_)13+WB{klcW*BI`y1XUb&vw{XDdY@ z{it-lm1um--pR$g6MCtm^)iFvFvYW<=1aaWT7Rb0ep(*XL=9wodyGmb!ZXBfYb)rc zaU8Yq`lA0cIs1ugsxLi6BX6rK*BFZYlFxsjD;JtQ+^ zv}sdf6^=kHqQ=F<9I4i^?E>JP&rkGw5L!pF6urR5gx;L9RR zy-lU3;*ff8qO(wQ{k%wjMM1O9@>vY!%AiJqbB;_Ft(mPNG+y4#jy`nEWe_ESMIzcC zmCpF=Ca%ga=-;BgpQ`eaaQ?FAW4u^V9*Pn>Pw2>OW7I;(H9D|J-OHZWgR*X?YAw%L z-C>yK-Ua+O6o@4Jj^TG5LJuSH_6Df&v1o*MWYb|0Kq$b|v(2ShYwL%DJ{fTkkBW3e zU4nP74P~50-;1_L$2zp`dM*uhb6d}xH6eK3Tkcm2)Pu|LmoezRR;k&K?#kF3XUOI| zk&b5e6=tseq)kcKt+}rZug+TjL@&p#dl^@n95wMOz2ln=T?H2hncSi(FQb1I_Ywc1 zJvf6-sKxtnUd+CDwLhpeu!-+q@W9IUZy^YA_pq@F850D&0`=)odk!$6G_nIF2sj;&pvxf>NAuG zR1X|gQ!uU06Oo(ortnZX_u0c=V=i4FGuOVd0_Eu5Df!dO-W&2bRKXXN-7cL*CRU=Jp zlJGy}DFGrxd9h+JrSYxps3{J+z;K-?r1zTcn~M4C)T#19@5Q<%Hay+al4SyS(fyXx z`-@0?tcLnTu=->dHa^dN1t<}sKef;A*0%)OOrcXfzli2(?TE{^9am8UpU@9q>cp6N z{hC+Dd(AJaZ~~Y`2J={NyeI=J)m0CR?<&p(8=Bd)*2f;2*Kk|5>^pQ#^~x>7ar2Y~ zp`qTJdwOy1BhR@kmKs7SbLq4*qCs;}n$XCXU{Vz{hLeKL7PxN@?B*JrXks&cl@olk zQ($WR;#kA0gQ-*PI)ukN;a2>0HX2kxcC|WIdb@Q;0CTGF3^S(++fLclv^a}(0q;?m z^w~U=ZpHP()SWmnm2b;RCOtW^3KC8!s61cky}&XaFb1xF0FR%>FAt_AC2EdADpu0$ z263`v_aUgUMN}=uHq9xmbS-l5d~9WcGC2_3xR^tXJ5wON-;l zmRG-FnVHSUQf$L9Iwg*h`hzwTkdLOtOnnUl{*<5X0it0d&EEj+_p*|p>}@1D)rU+4 z`^&bqle(9xA;dr?t>n}6792V*$`iTff>|bebfyKeT z1y}CTjXQ%1_B-b;8iZ_qwzE!Twst`1RcdST|J0>gYwRJ{^~`LUc`WI=6Bx0Z$KS@* zEtgY0Im`A5tECVie=Y_fmDa>qmdL;@Sf#OmP08eh5!IS zdk>z8YecsEZ4EMe*5Nn+Y$zNsBl~JJ_BY-FPgztjri5-xcU&gmoMBIm0AOaz~}akAOBLp{k&SzcFel=m9rQ zfMpp~$BMB6tOYo@uV&CK`@jOIO)$CWz8MB$km3WF2^&aJqWe|@_#x!zJfLryH1p@P z;Jl|I;MdGUP`f}0otxO$nG0&a7-o5xxpP162%fHK%l<3bc0UBIY6=_KJk23yaU zErbY(7*cecrenV9Wp}~7bTAwAzYQrafO|IldEVBjiGFY#DW=rY7P$rdf0#ouZeAI! z+76)U`!K29f4aVY?t?|6o{b}2&DX6x5>mG9d2ru0$uVMNaqwl`2M5|=7|K$LKR-** z{U1Pu#@F}Si4^vo3;DXVUz??e_u%_K6(@}yWkW-nbvo{nFk}8>_SOPVPDW!6Ck(+E9X z4?7-t@~i%~)~iF^UW>kh9a*|HQsV9}BIMIwR1PWgLUtOOJXpvndnD|qx1oH!N|kgu*a<4axI^1|KaKZKz-?|a&b(&H z0US$^8BoCu4aTQIVXN)28IZ`~a3iTmRH^#HHp?de2P1;Q)_t&l%%-!W&?gBx%fCR} zuVWR_Z)4I&IaiuSy4|?bY?F-xdj8KGx!+C?JYS3bam!YF<8gU0m(HDkN_1O%#z9S0 z6-ng&M=J2mM|m4+wkk%}Hfn_$UvY4I!8F^0^}g$$xWF6 diff --git a/docs/design/cross_config_implementation_report.md b/docs/design/cross_config_implementation_report.md new file mode 100644 index 00000000..74b175e1 --- /dev/null +++ b/docs/design/cross_config_implementation_report.md @@ -0,0 +1,185 @@ +# Cross-Configuration Alignment Implementation Report + +Status: V1 framework snapshot, 2026-07-19 + +Related work: + +- [Roadmap #83](https://github.com/RL-Align/RL-Kernel/issues/83) +- [Cross-configuration alignment #111](https://github.com/RL-Align/RL-Kernel/issues/111) +- [Numerical contract #108](https://github.com/RL-Align/RL-Kernel/issues/108) +- [V1 contract](cross_config_logprob_drift_contract.md) + +## Result and claim boundary + +This change provides a small framework for planning and executing paired +rollout/training logprob comparisons across controlled configuration changes. It +includes strict configuration loading, bounded case planning, exact semantic +operator selection, lifecycle-aware runtime materialization, paired read-only +scoring, fixed-contract comparison, append-only artifacts, and validated resume. + +The included executable path is deliberately CPU-only. It validates framework +plumbing with a synthetic model and temporary selected-logprob backends; it does +not claim production vLLM, FSDP, TP, CP, accelerator, or distributed numerical +alignment. The S1, S2, and S3 examples are plans, not execution evidence. + +## Architecture + +The implementation keeps configuration, operator resolution, runtime ownership, +and execution separate: + +```text +JSON -> ExperimentConfig -> Planner -> ExperimentPlan + | + build_execution_plan + | + operator-bound ExecutionPlan + / \ + operator session RuntimeMaterializer + | + RuntimeBinding + | + ArtifactStore <- PairedRunner -> fixed comparator +``` + +| Boundary | Responsibility | +| --- | --- | +| `config.py` and `planner.py` | Load strict, versioned JSON; normalize the ten supported knobs; emit an `ExperimentPlan` containing a baseline plus declared OAT or explicit pairwise cases; compute stable semantic case IDs without importing a runtime. | +| `build_execution_plan` | Resolve rollout/training selections, bind them into immutable case identity, and emit canonical operator-bound `ExecutionPlan` rows shared by planning and execution. | +| `SemanticOperatorCatalog` | Store immutable backend descriptors and their target, device, dtype, topology, lifecycle, factory, and observability constraints. | +| `OperatorSession` | Resolve and instantiate exact rollout/training implementations for one case, cache only within that case, and produce concrete provenance. | +| `RuntimeMaterializer` | Apply each normalized knob through an owning adapter and report requested, materialized, and actual values with status and lifecycle evidence. | +| `RuntimeBinding` | Carry only backend-neutral batch, side-configuration, topology, scorer, operator-backend, and runtime-kind mappings. Runtime-specific engine objects stay behind the adapter boundary. | +| `PairedRunner` | Supervise isolated rollout/training scoring children, enforce timeout and read-only model state, validate ranks and exact operator instances, compare selected logprobs, and coordinate resume/publication. | +| `ArtifactStore` | Publish immutable attempt directories and write `COMPLETE` last. Resume accepts only an attempt whose identity, execution, provenance, tensors, and comparison still validate. | + +Runtime bindings deep-freeze their execution handoff. Rollout topology contains +only rollout-owned world/TP/CP state, while training topology contains only +training-owned world/sharding state; neither side is padded with fields owned by +the other. + +The fixed-threshold comparator applies the repository numerical contract only to +active selected tokens. Identity violations, invalid artifacts, non-finite +scores, and zero active tokens cannot become passes. Diagnostics remain separate +from the pass/fail rule. + +## Configuration and operator selection + +Execution controls do not live in experiment JSON. `scenario` is metadata; the +CLI chooses planning versus execution, the runtime adapter, temporary-operator +authorization, timeout, and resume policy. + +`logp.backend` is the concise choice when rollout and training use the same +selected-logprob implementation. The optional top-level `operators` mapping is +the extension point for independent sides and per-backend options: + +```json +{ + "baseline": { + "logp": {"backend": "rlkernel.reference_logp"} + }, + "operators": { + "selected_logprob": { + "rollout": "rlkernel.reference_logp", + "training": { + "backend": "vendor.training_logp", + "options": {"mode": "exact"} + } + } + } +} +``` + +The explicit rollout backend must agree with the baseline `logp.backend`. +Explicit operators cannot be combined with `logp.backend` interventions because +that would make the planned knob differ from the implementation actually used. +Unknown fields, threshold overrides, hidden execution controls, duplicate JSON +keys, and non-finite values are rejected. + +## CLI + +Planning validates and records a plan without constructing a runtime: + +```bash +python -m rl_engine.alignment.cross_config plan \ + examples/cross_config_s3_qwen3_8b_tp4_cp4_bf16.json +``` + +The only shipped execution adapter is the explicit CPU smoke runtime: + +```bash +python -m rl_engine.alignment.cross_config run \ + examples/cross_config_s0_cpu_smoke.json \ + --runtime cpu-smoke \ + --allow-smoke-operators +``` + +Both commands accept `--output-root`. `run` also exposes a per-attempt timeout +and `--no-resume`; these policies are intentionally absent from the JSON schema. + +## CPU testing adapter + +`rl_engine.alignment.testing.cpu_cross_config` owns the synthetic causal model, +stateless CPU scorer, `CpuSmokeMaterializer`, canonical batch construction, and +CPU experiment helpers. Keeping these objects outside the core package prevents +test hardware and model assumptions from becoming runtime abstractions. + +Temporary selected-logprob implementations live under +`rl_engine/alignment/testing/smoke_ops`. They advertise CPU as their only device, +are not registered by default, and require explicit policy authorization. The +reference backend performs `log_softmax` plus gather; the offset backend is used +only by focused mismatch tests. The shipped S0 example selects reference on both +sides and contains one baseline case. + +## Scenario evidence + +| Scenario | Planned cases | Current evidence | +| --- | ---: | --- | +| S0 CPU framework smoke | 1 | One reference/reference case passes on CPU; a second invocation validates and resumes the same complete attempt. | +| S1 distributed smoke | 5 | Configuration loading and planning only. | +| S2 vLLM TP versus FSDP | 10 | Configuration loading and planning only. | +| S3 Qwen3-8B TP=4, CP=4, BF16 | 11 | Configuration loading and planning only. | + +Planning success does not imply that the requested production topology can be +materialized. Unsupported or unobservable runtime settings fail strict execution +instead of silently falling back. + +## Validation snapshot + +```text +Focused contract/runtime/runner/CLI and existing regression tests: +60 passed in 2.20s + +CPU-collectable repository suite: +418 passed, 242 skipped in 15.83s + +Named scenario checks: +S0 run: 1 pass, then 1 validated resume +S1/S2/S3 plan: 5 / 10 / 11 cases, no runtime constructed +``` + +The final review also checks JSON syntax, formatting, static typing, strict +documentation build, and `git diff --check`. + +The full CPU command excludes `test_grpo_loss.py` and `test_ratio_kl.py` +because those modules require Triton during collection. It ran outside the +restricted sandbox so Gloo and POSIX shared-memory tests could use host +resources. + +## Extension path and known gaps + +A production backend extends the semantic catalog with a descriptor and factory, +then supplies injection/read-back hooks through its runtime adapter. The planner +and runner do not need backend-specific branches. Operator correctness remains +owned by the operator implementation; the framework verifies exact selection, +materialization evidence, paired identity, comparison, and provenance. + +Production execution still requires: + +- verified selected-logprob injection and read-back for the rollout engine; +- a read-only pre-update training scorer for FSDP and distributed rank evidence; +- context-parallel application/read-back and process-group orchestration; +- accelerator-backed lifecycle and cleanup tests; and +- the production kernels tracked by their owning workstreams. + +Until those adapters exist, S1-S3 remain reproducible planning inputs and the CPU +smoke remains a framework claim only. diff --git a/docs/design/cross_config_logprob_drift_contract.md b/docs/design/cross_config_logprob_drift_contract.md new file mode 100644 index 00000000..3c1750a7 --- /dev/null +++ b/docs/design/cross_config_logprob_drift_contract.md @@ -0,0 +1,306 @@ +# Cross-Configuration Logprob Drift Contract + +Status: V1 implementation contract + +Related work: + +- [Roadmap #83](https://github.com/RL-Align/RL-Kernel/issues/83) +- [Cross-configuration alignment #111](https://github.com/RL-Align/RL-Kernel/issues/111) +- [Numerical contract #108](https://github.com/RL-Align/RL-Kernel/issues/108) + +## Goal and boundary + +This framework isolates configuration changes that can make rollout-selected +log probabilities differ from training-side recomputation. It provides typed +plans, lifecycle-aware runtime materialization, exact semantic-operator +selection, paired read-only scoring, append-only artifacts, and safe resume. + +It does not implement production AG, RS, GEMM, attention, logprob, TP-invariant, +CP-aware, or deterministic collective kernels. Those implementations remain +owned by their operator workstreams and integrate through the semantic operator +catalog described below. + +## The only pass/fail rule + +For every active selected response/action token: + +```text +abs(training_logprob - rollout_logprob) > fixed_threshold +``` + +The fixed threshold is loaded from the repository numerical contract. It is not +a config field, CLI flag, experiment axis, workload policy, or operator option. +Equality with the threshold is not a mismatch. + +```python +mismatch_mask = active_mask & ( + torch.abs(training_logprobs - rollout_logprobs) > fixed_threshold +) +``` + +Mean, percentiles, maximum absolute difference, mismatch ratio, worst-token +location, and approximate KL are diagnostics only. They never change pass/fail. +The token-level artifact persists both logprob tensors, the active mask, and the +resolved threshold so the mask can be recomputed offline. + +Zero active tokens produce `ZERO_ACTIVE_TOKENS`, never a pass. Non-finite or +non-floating active scores produce `INVALID_ARTIFACT`. + +## Identity before numerics + +A comparison is valid only when both scorers use the same logical input: + +- immutable checkpoint and model version; +- tokenizer ID and tokenization policy; +- generated token IDs and selected-token IDs; +- active and attention masks; +- pre-update model state; +- required position, cache, and packing metadata. + +The training scorer teacher-forces the already generated sequence. It cannot +generate replacement tokens, use a KV cache when the frozen identity forbids +one, own an optimizer, update parameters or buffers, or leave the model in a +different mode. An identity violation is `INVALID_IDENTITY`, not numerical +drift. + +## V1 configuration + +The user supplies one explicit baseline plus declared interventions. Lists do +not imply a Cartesian product. + +```json +{ + "schema_version": "cross_config.experiment_config.v1", + "experiment_id": "qwen3-8b-alignment", + "scenario_id": "qwen3-8b-tp4-cp4-bf16", + "contract_source": "ws1", + "contract_version": "current", + "strategy": "one_at_a_time", + "strict_fallback": true, + "identity": {"...": "frozen scoring identity"}, + "baseline": { + "batch": {"size": 8}, + "rollout": { + "tensor_parallel_size": 4, + "context_parallel_size": 4, + "dtype": "bfloat16", + "enable_prefix_caching": true, + "enforce_eager": false + }, + "training": { + "attention_backend": "flash_attention_2", + "compute_dtype": "bfloat16", + "sharding": "fsdp" + }, + "logp": {"backend": "native"} + }, + "interventions": [ + {"path": "batch.size", "values": [1]}, + {"path": "logp.backend", "values": ["rlkernel.reference_logp"]} + ], + "scenario": {"level": "S3", "device": "cuda"} +} +``` + +`scenario` is metadata. Execution mode and authorization policy belong to the +CLI, so a config cannot hide `plan_only`, expected test outcomes, or permission +to activate temporary operators. + +### Exact knob allowlist + +| Knob | Minimum lifecycle | Meaning | +|---|---|---| +| `batch.size` | request | Canonical sample chunking only. | +| `rollout.tensor_parallel_size` | process | Rollout TP world. | +| `rollout.context_parallel_size` | process | Rollout CP world. | +| `rollout.dtype` | engine construction | Rollout numerical dtype. | +| `rollout.enable_prefix_caching` | engine construction | Engine cache policy. | +| `rollout.enforce_eager` | engine construction | Eager versus optimized/graph path. | +| `training.attention_backend` | engine construction | Training scorer attention implementation. | +| `training.compute_dtype` | engine construction | Training scorer compute dtype. | +| `logp.backend` | engine construction | Both-sides selected-logprob shortcut. | +| `training.sharding` | process | Training topology, such as unsharded or FSDP. | + +TP/vocabulary layout is derived and recorded, not user-settable. Tokenization, +masks, positions, checkpoint identity, and pre-update state are invariants, not +ordinary knobs. Quantization, FP8, MoE, speculative decoding, pipeline +parallelism, and arbitrary runtime fields are deferred. + +### Planning + +`one_at_a_time` emits one baseline and cases that change exactly one declared +path. `pairwise` is opt-in and expands only explicitly listed path pairs. The +planner normalizes aliases, validates the allowlist and capability constraints, +and reports structured issues without creating engines. + +Stable case IDs hash normalized requested values, identity, contract version, +and scenario definition. Runtime readback never rewrites a case ID. A retry gets +a new attempt ID under the same case. + +## Architecture and extension points + +The core has one-way responsibilities: + +```text +strict config -> Planner -> ExperimentPlan -> build_execution_plan + | + operator-bound ExecutionPlan + | + runtime adapter -> RuntimeBinding + | + paired runner + / \ + artifact store comparator +``` + +- `config.py` owns the external schema and strict JSON loading. +- `schema.py` owns immutable, versioned domain records. +- `planner.py` owns normalization, the knob catalog, OAT, and pairwise cases. +- `execution_plan.py` binds the selected rollout/training operators into each + immutable case and produces canonical rows shared by planning and execution. +- `runtime.py` owns the adapter protocol, three-stage materialization, lifecycle + fingerprints, and a backend-neutral execution binding. +- `comparison.py` owns identity validation and the fixed-threshold result. +- `runner.py` coordinates paired execution and atomic publication; private + execution, provenance, and resume modules isolate process supervision and + validation details. +- `artifacts.py` owns append-only attempts and resume discovery. +- `semantic_registry.py` owns generic operator descriptors and case-local + resolution sessions; it is shared by future alignment features. + +The package root exposes only the common planning and execution facade. Runtime, +artifact, schema, and operator internals remain in their owning modules. + +### Runtime adapters + +A runtime adapter receives the normalized case and returns one application +record per knob: + +```text +requested -> materialized -> actual +``` + +Each record includes status (`applied`, `fallback`, `unsupported`, +`unobservable`, or `error`), evidence, and lifecycle. The facade derives +construction, distributed-context, and process fingerprints. Reuse is allowed +only when all relevant fingerprints and operator bindings match. + +Adapters may construct repository-native vLLM, training, or stateless config +objects internally. The core runner receives only backend-neutral batch, side +configuration, topology, scorer, operator-backend, and runtime-kind mappings, +so adding a runtime does not add branches to the planner or runner. + +Strict execution rejects fallback, ignored settings, unobservable critical +values, stale registry state, and incompatible reuse. Fallback is measurable +only when it is itself the declared intervention. + +## Semantic operator selection + +The first semantic operator is `selected_logprob`. Rollout and training can +select implementations independently: + +```json +{ + "operators": { + "selected_logprob": { + "rollout": "rlkernel.reference_logp", + "training": { + "backend": "rlkernel.reference_logp", + "options": {} + } + } + } +} +``` + +When `operators` is absent, `logp.backend` selects the same backend on both +sides. An explicit mapping is bound into execution identity before a runtime is +created. A `logp.backend` intervention cannot be combined with a fixed explicit +mapping because that would create a knob that no longer changes execution. + +Each backend descriptor declares: + +- semantic operation and backend ID; +- supported target tags, devices, dtypes, and topology values; +- alignment properties and lifecycle; +- implementation factory and version/build fingerprint; +- explicit fallback policy and temporary-test marker. + +`SemanticOperatorCatalog` stores immutable descriptors. Each case creates an +`OperatorSession` for resolution, instantiation, caching, and provenance. Failed +or cached state cannot leak into the next case. Strict resolution never invokes +legacy priority fallback. + +Adding a production implementation requires its existing semantic interface, +one descriptor, runtime injection hooks where needed, operator-owned correctness +tests, and one framework case. It does not require a planner change. + +## Artifacts and resume + +Attempts are append-only: + +```text +runs// + experiment.json + plan.jsonl + cases/// + requested.json + materialized.json + actual.json + identity.json + score_rollout.pt + score_training.pt + comparison.json + token_diffs.pt + COMPLETE +``` + +`COMPLETE` is published last. Resume accepts only a complete attempt whose case, +identity, materialization, scorer, operator, environment, and tensor artifacts +match the current execution key. Partial, malformed, or tampered attempts are +ignored; an older valid attempt may still be reused. Existing files are never +overwritten. + +## CPU smoke boundary + +The only executable adapter delivered here is under +`rl_engine.alignment.testing.cpu_cross_config`. It is explicitly CPU-only and +uses a deterministic synthetic model plus read-only stateless scoring. Named +distributed and accelerator scenarios are configuration/plan coverage only. + +Temporary selected-logprob backends live together under +`rl_engine/alignment/testing/smoke_ops`: + +- `smoke_only.logp_reference`: PyTorch `log_softmax` plus gather; +- `smoke_only.logp_offset`: the same result with an authorized deterministic + offset used to prove mismatch detection. + +They are CPU-only, marked `is_smoke_only`, unregistered by default, and require +both explicit registration and execution policy authorization. Their exact +removal procedure is in `SMOKE_OPERATORS.md`. Remove them when equivalent +production operators pass the same framework cases, then remove the opt-in flag +and temporary test marker. + +## Scenario levels and claims + +| Level | Purpose | Current claim | +|---|---|---| +| S0 | Local CPU framework smoke | Executable: config, planner, operator selection, paired scoring, comparison, artifacts, resume. | +| S1 | Small distributed lifecycle smoke | Plan only until suitable hardware/runtime adapters exist. | +| S2 | Named vLLM TP versus training FSDP comparison | Plan only. | +| S3 | Qwen3-8B TP=4, CP=4, BF16 milestone | Plan only; no production alignment claim. | + +Run the shipped examples with: + +```bash +python -m rl_engine.alignment.cross_config plan \ + examples/cross_config_s3_qwen3_8b_tp4_cp4_bf16.json + +python -m rl_engine.alignment.cross_config run \ + examples/cross_config_s0_cpu_smoke.json \ + --runtime cpu-smoke \ + --allow-smoke-operators +``` + +Passing S0 proves framework plumbing only. It does not prove accelerator, +distributed, production-operator, or roadmap numerical alignment. diff --git a/docs/design/ws2_cross_config_logprob_drift_contract.md b/docs/design/ws2_cross_config_logprob_drift_contract.md deleted file mode 100644 index 3d7fac06..00000000 --- a/docs/design/ws2_cross_config_logprob_drift_contract.md +++ /dev/null @@ -1,874 +0,0 @@ -# WS2 Cross-Config Logprob Drift Contract - -Status: RFC - -Tracking issues: - -- [#111: WS2 cross-config alignment](https://github.com/RL-Align/RL-Kernel/issues/111) -- [#108: WS1 numerical contract](https://github.com/RL-Align/RL-Kernel/issues/108) - -## Motivation - -WS2 covers rollout and training paths that use different parallelism strategies, such as -rollout tensor parallelism and training FSDP. The alignment problem is not a single-op -accuracy check. It is end-to-end floating-point drift across tokenizer, masks, serving, -rollout, and training recomputation before any optimizer update. - -For PPO, GRPO, and related RL post-training algorithms, the most direct pre-update signal -is selected-token log probability drift. If rollout-side `old_logprobs` and train-side -recomputed log probabilities disagree for the same checkpoint, same token ids, same masks, -and same model version, classify the failure as infrastructure, precision, mask, -tokenizer, or serving-path drift. Do not classify that failure as an algorithm or reward -problem until pre-update logprob alignment is clean. - -Aggregate KL-style diagnostics are useful but not sufficient as the primary WS2 contract. -In training-inference mismatch cases, KL estimates can stay flat or fail to expose the -early failure phase, because the first-order issue is token-level rollout-vs-training -probability disagreement before the optimizer update, not necessarily a large aggregate -policy-space shift. - -## Framework Upgrade Overview - -The upgrade moves cross-config validation from two separately configured execution paths -that require a manual comparison into a shared runtime flow. `RuntimeTools` coordinates -the rollout and training configurations, while `PairedRunner` collects their selected -logprobs and performs the comparison automatically. This keeps both sides aligned on the -same inputs and makes drift visible as part of the run rather than as a follow-up manual -check. - -![Before and after framework upgrade](../assets/ws2-cross-config-before-after.png) - -The left side shows the pre-upgrade flow, where `VLLMSamplerConfig` and -`TorchRLTrainingConfig` feed independent executors and the results are manually compared. -The right side shows the upgraded flow, where `RuntimeTools` and `PairedRunner` connect the -two paths and produce an automatic comparison while preserving the shared -`KernelRegistry` contract. - -## Scope - -This RFC defines what WS2 cross-config alignment measures, how failures are classified, -and the modular implementation roadmap for making that contract executable. It does not -itself add a test harness, distributed tests, runtime gates, layer-wise probes, or -distributed fixes. - -Out of scope for this document: - -- Implementing multi-GPU test infrastructure. -- Adding runtime pass/fail gates. -- Adding automatic layer-wise drift probes. -- Fixing TP, FSDP, SP, cache, mask, tokenizer, or serving-path bugs. -- Defining a second numerical tolerance table. -- Reimplementing work owned by the adjacent TP, SP, collective, training-integration, or - layer-probe issues referenced by the roadmap below. - -## Measurement Contract - -The primary metric is selected-token logprob drift: - -```text -dlogp = train_recomputed_logp - rollout_old_logp -``` - -Compute `dlogp` only on active response/action tokens. Prompt tokens, padding tokens, and -masked-out response positions are excluded from every aggregate metric. - -The comparison must use teacher-forcing scoring on the training side. The scored sequence -is the already-sampled rollout sequence; the training path must not resample or regenerate -tokens for this contract. - -The rollout and training values are comparable only when they share the same logical -inputs: - -- Same checkpoint and same model version. -- Same input token ids. -- Same selected response/action token ids. -- Same attention mask and action mask. -- Same tokenizer version and tokenization policy. -- Same padding layout semantics, including left-padding or right-padding behavior. -- Same pre-update state, before any optimizer step, weight sync, or policy mutation that - belongs to the next training step. - -If the implementation has explicit position ids, cache-position metadata, sequence ids, or -packed-sequence metadata, those inputs are part of the comparison contract as well. - -## Primary Failure Signal - -The pass/fail decision starts from `dlogp` over active tokens. Reward, gradnorm, -weightnorm, and update norm are downstream symptoms. They are useful for debugging and -triage, but they are not the primary contract for cross-config alignment. - -The zero-update expectation is: - -```text -train_recomputed_logp ~= rollout_old_logp -ratio0 ~= 1 -approx_kl0 ~= 0 -``` - -The acceptable meaning of `~=` is defined by the WS1 per-dtype numerical threshold table -from [#108](https://github.com/RL-Align/RL-Kernel/issues/108). This RFC defines the -measurement surface and classification rules only. - -## Diagnostics - -All diagnostics are computed on active response/action tokens only. - -| Metric | Definition | Purpose | -| --- | --- | --- | -| `ratio0` | `exp(dlogp)` | Zero-update policy ratio implied by train-vs-rollout logprob drift. | -| `clipfrac0` | Mean indicator that `ratio0` falls outside the configured PPO/GRPO clip range. | Detects whether drift alone would trigger clipping before any update. | -| `approx_kl0` | Masked mean of `exp(dlogp) - 1 - dlogp`. | Zero-update approximate KL implied by logprob drift. | -| `mean_abs_dlogp` | Mean of `abs(dlogp)`. | Average selected-token drift. | -| `p95_abs_dlogp` | 95th percentile of `abs(dlogp)`. | Tail drift below outliers. | -| `p99_abs_dlogp` | 99th percentile of `abs(dlogp)`. | High-tail drift. | -| `max_abs_dlogp` | Maximum of `abs(dlogp)`. | Worst selected-token mismatch. | - -When the run is distributed, report optional per-rank versions of the same metrics. The -per-rank view should preserve enough metadata to identify the rollout rank, training rank, -parallelism mode, dtype, padding side, cache mode, and local active-token count for that -rank. - -## Tolerance Source - -This RFC does not define a separate numerical tolerance table. The single source of truth -for acceptable numerical drift is the per-dtype threshold table owned by -[#108](https://github.com/RL-Align/RL-Kernel/issues/108). - -For WS2, acceptable numerical drift means that `max_abs_dlogp` over active -response/action tokens satisfies the WS1 per-dtype threshold from #108. If the #108 table -changes, WS2 inherits that policy without editing this document or maintaining a second -table. - -## Tolerance Interpretation and Effect-Based Validation - -Numerical tolerances in this RFC are infrastructure contract thresholds, not a universal -statement of algorithmic harmlessness. There is no model-independent scale that proves a -given train-vs-rollout logprob difference is harmless for every algorithm, reward model, -prompt distribution, sequence length, or optimization schedule. Any hand-written threshold -encodes a prior about acceptable numerical error. WS2 therefore does not introduce an -additional algorithmic noise budget, nor does it define a new estimator for tolerable -logprob noise. - -The #108 threshold defines whether rollout and training paths are numerically aligned -enough to continue debugging the failure as an algorithmic or reward problem. It does not -prove that all smaller drift is behaviorally irrelevant, and it does not imply that all -larger drift is the only cause of downstream failure. - -When downstream model-effect validation is available, such as reward trajectory, train KL, -eval win rate, collapse rate, policy regression tests, or task-specific success metrics, -use it as a severity and root-cause prioritization signal. It must not replace the -pre-update selected-token logprob contract. A run can be numerically out of contract even -if a short downstream run appears healthy, and a run can be numerically in contract while -still failing because of algorithmic tuning, reward hacking, insufficient KL control, or -data issues. - -The intended interpretation is: - -```text -#108 per-dtype threshold: - numerical infrastructure contract - -selected-token dlogp: - primary WS2 train-vs-rollout drift surface - -downstream model effect: - practical severity and algorithmic relevance signal - -KL / ratio / percentile diagnostics: - debugging and triage signals, not replacement pass/fail criteria -``` - -## Drift Source Taxonomy - -Before treating train-vs-rollout drift as generic algorithmic noise, WS2 should classify -likely sources of mismatch. At minimum, the following source classes should be considered -separately. - -### Arithmetic Schedule Drift - -Arithmetic schedule drift comes from different floating-point operation order between -rollout and training. This includes different kernels, fused vs unfused implementations, -compiler-generated graph rewrites, attention implementation differences, matmul epilogue -differences, accumulation dtype differences, and changes introduced by advanced compilers -or graph optimizers. - -This class answers the question: - -```text -Do rollout and training compute mathematically equivalent expressions using different -floating-point schedules? -``` - -Examples include: - -- Fused attention vs unfused attention. -- Different FlashAttention or SDPA backends. -- Fused RMSNorm or LayerNorm vs decomposed normalization. -- Compiler-reordered graph segments. -- Different matmul epilogues or activation fusion. -- Different accumulation precision in otherwise equivalent kernels. - -### Reduction and Collective Drift - -Reduction drift comes from operations whose floating-point result depends on reduction -order, parallel topology, or concurrent execution. This includes local reductions, -cross-rank reductions, all-reduce, reduce-scatter, gather/scatter patterns, sharded logits -or loss computation, tensor-parallel collectives, FSDP reductions, and nondeterministic -reduction scheduling. - -This class answers the question: - -```text -Does the mismatch appear because rollout and training aggregate partial results in -different orders or across different rank topologies? -``` - -Examples include: - -- TP logits produced through a different collective path from the training path. -- FSDP reduce-scatter or all-gather changing accumulation order. -- Per-rank partial reductions with different shard boundaries. -- Loss or logprob reductions performed before vs after cross-rank communication. -- Nondeterministic collective algorithms or concurrent reductions. - -### Quantization and Dequantization Drift - -Quantization drift comes from representing weights, activations, KV cache, logits, or -intermediate tensors with different quantization policies between rollout and training. -Quantization is not merely a floating-point ordering issue; it introduces representation -noise through scales, zero points, clipping, grouping, calibration, and dequantization -paths. - -This class answers the question: - -```text -Does the mismatch appear because rollout and training use different numerical -representations or quantization policies? -``` - -Examples include: - -- Rollout uses weight-only quantization while training recomputation uses bf16/fp16 - weights. -- Different quantization group sizes. -- Different activation quantization or KV-cache quantization policy. -- Different scale computation or calibration data. -- Different dequantization placement relative to fused kernels. -- Serving-path quantization that is absent from the training path. - -### Logical Input and Metadata Drift - -Logical input mismatch must be ruled out before interpreting any result as numerical -drift. This class includes tokenizer version, tokenization policy, attention mask, action -mask, padding side, explicit position ids, cache positions, sequence ids, packed-sequence -metadata, and serving-path request formatting. - -This class answers the question: - -```text -Are rollout and training actually scoring the same logical sequence under the same masking -and positional semantics? -``` - -If this class is not clean, the comparison is invalid rather than merely noisy. - -## Decision Rule - -Use this order when classifying a cross-config failure: - -1. If pre-update selected-token logprobs do not match under the same checkpoint, same token - ids, same masks, and same model version, treat the failure as infrastructure, - precision, mask, tokenizer, or serving-path drift. -2. If `max_abs_dlogp` violates the #108 threshold but downstream metrics look healthy in a - short run, keep the issue classified as infrastructure drift. Short-horizon model - health does not prove the drift is safe. -3. If KL or ratio diagnostics move before gradnorm or update norm moves, treat the failure - as likely infrastructure or logprob plumbing. -4. If gradnorm or update norm moves first and KL moves later, treat the failure as more - likely algorithmic tuning, such as learning rate, KL beta, reward scale, or advantage - outliers. -5. If only some ranks drift, treat the failure as distributed infrastructure until rank - placement, shard boundaries, collective algorithms, local active-token counts, masks, - and cache-position issues are ruled out. -6. If reward rises and then collapses while pre-update logprob alignment is clean, treat - the failure as more likely algorithmic, reward hacking, data-related, or insufficient - KL constraint. - -This classification does not prove root cause by itself. It defines the first branch in -the debugging tree so WS2 bugs do not get misfiled as reward or algorithm regressions -before the zero-update logprob contract is satisfied. - -## Layered Ablation Strategy - -WS2 should not treat train-vs-rollout mismatch as a single undifferentiated error source. -Later tests should use a layered ablation strategy that changes one source class at a time -whenever the implementation allows it. - -The minimum useful ablation structure is: - -```text -A0. Fully aligned reference - Same checkpoint, same dtype policy, same kernels where possible, same reduction - topology where possible, same quantization policy, same tokenizer, same masks, same - padding, same cache/position metadata. - -A1. Arithmetic-schedule-only mismatch - Keep logical inputs, reduction topology, and quantization policy aligned. Allow only - kernel, fusion, compiler, or graph execution differences. - -A2. Reduction-topology-only mismatch - Keep logical inputs, kernel policy, and quantization policy aligned. Allow only - reduction order, collective topology, sharding, or rank placement differences. - -A3. Quantization-only mismatch - Keep logical inputs, kernel policy, and reduction topology aligned. Allow only - quantization, dequantization, scale, group size, or representation differences. - -A4. Pairwise mismatches - Enable two mismatch classes at a time: - arithmetic + reduction - arithmetic + quantization - reduction + quantization - -A5. Full production mismatch - Use the real rollout and training configurations, including all production - differences. -``` - -Each ablation should collect the same primary and diagnostic metrics: - -```text -primary: - dlogp over active response/action tokens - max_abs_dlogp - -diagnostics: - mean_abs_dlogp - p95_abs_dlogp - p99_abs_dlogp - ratio0 - clipfrac0 - approx_kl0 - per-rank versions when distributed - -metadata: - dtype - kernel/backend choices - fusion/compiler mode - reduction/collective topology - quantization policy - padding side - cache mode - position/cache-position metadata - active-token count -``` - -When downstream model-effect validation is available, the same ablations should also -record practical training outcomes, for example reward trajectory, training KL, entropy, -clip fraction, update norm, collapse rate, and task-specific evaluation metrics. These -downstream metrics are not the WS2 pass/fail contract, but they help rank which numerical -mismatch class matters most for the workload. - -## Ablation Interpretation Rules - -Use these rules when reading the ablation matrix: - -1. If the fully aligned reference fails, the issue is not a cross-config mismatch yet. - First debug the base scoring path, masks, tokenizer, position metadata, checkpoint - identity, or implementation correctness. -2. If a single-source ablation fails the `max_abs_dlogp` contract, that source class is - sufficient to create unacceptable train-vs-rollout drift under the tested workload. For - example, if only quantization is misaligned and the run fails, quantization is a - dominant source candidate for that task and configuration. -3. If all single-source ablations pass, but pairwise or full-production mismatches fail, - the failure is likely an interaction effect. Identify the minimal failing pair before - attributing the issue to any single subsystem. -4. If one single-source ablation passes the numerical contract but shows materially worse - downstream model effect, record it as behaviorally sensitive even if it remains - numerically in contract. This is a signal that the #108 infrastructure tolerance may be - sufficient for numerical alignment but not necessarily predictive of algorithmic - robustness for that workload. -5. If pre-update logprob alignment is clean but downstream training still collapses, - classify the failure as more likely algorithmic, reward-related, data-related, or - KL-control-related rather than cross-config numerical drift. - -## Minimal and Layered Alignment Principle - -The governing principle of WS2 is **minimal alignment**: - -> Keep rollout and training semantically identical, then align only the smallest numerical -> layer needed to satisfy the selected-token logprob contract. - -WS2 does not require every internal tensor, kernel, reduction, or execution schedule to be -identical. If the production rollout and training paths already satisfy the #108 -`logprob` tolerance, no numerical alignment change is required. Different engines are -allowed to keep different high-performance implementations. - -Minimal alignment does not relax logical correctness. Checkpoint/version, token ids, -masks, tokenizer semantics, and required position metadata must match exactly. A logical -input mismatch invalidates the experiment; it is not acceptable numerical drift. - -### Alignment Ladder - -Use the following ladder in order and stop at the first level that satisfies the contract: - -| Level | Action | Production implication | -| --- | --- | --- | -| L0: semantic identity | Make logical inputs and model version exactly comparable. | Mandatory for every case. | -| L1: observable contract | Keep both production paths unchanged and compare selected-token logprobs. | Stop here if #108 passes. | -| L2: source isolation | Change one declared knob at a time to locate the smallest sufficient drift source. | Diagnostic only; do not change production yet. | -| L3: local alignment | Align or fix one operator, collective, metadata field, or representation policy. | Preferred production fix when L1 fails. | -| L4: layered alignment | Align the smallest interacting pair or contiguous layer boundary that is required. | Use only when no single local change is sufficient. | -| L5: full/bitwise alignment | Force broad identical paths or reference implementations. | Diagnostic fallback, not the default WS2 exit criterion. | - -The chosen fix should minimize, in order: - -1. semantic scope changed; -2. number of aligned knobs; -3. performance and memory overhead; -4. engine-specific intrusion; -5. maintenance burden. - -A fix is incomplete if it proves only that the fully aligned reference passes. It must -also show that unrelated rollout/training differences can remain enabled. Conversely, WS2 -must not reject a configuration merely because internal tensors are not bitwise equal when -the selected-token contract passes. - -## Controller-Centered Design - -The central feature is an ablation controller, not a hard-coded list of distributed -tests. It separates experiment planning from engine-specific knob application. - -```mermaid -flowchart LR - Definition["ExperimentDefinition
identity + baseline + axes + constraints"] - Planner["GridPlanner
product / one-at-a-time / pairwise"] - Isolation["IsolationValidator
declared deltas only"] - Definition --> Planner --> Isolation - - Isolation --> Cases["ExperimentCase[]
stable ids + provenance"] - - subgraph Materializers["Knob materializers"] - Rollout["vLLM adapter"] - Training["stateless / FSDP adapter"] - Kernel["kernel policy adapter"] - Environment["process/build environment adapter"] - end - - Cases --> Materializers - Materializers --> Runner["isolated paired runner"] - Runner --> Samples["canonical alignment samples"] - Samples --> Comparator["identity validator + dlogp comparator"] - Comparator --> Cube["result cube
axes + per-rank reports + cost"] - Cube --> Analyzer["minimal sufficient alignment analyzer"] -``` - -### Core Objects - -The implementation should expose a small typed model rather than passing more loose -dictionaries through the current executors: - -- `SemanticIdentitySpec`: checkpoint/weight version, tokenizer, fixed token sequences, - masks, and position metadata that must match. -- `ScorerSpec`: rollout or training engine, world size, device/dtype, and immutable engine - construction settings. -- `KnobDefinition`: one controllable source of variation. -- `ExperimentDefinition`: baseline scorers plus axes, constraints, and measurement policy. -- `ExperimentCase`: one fully materialized grid point with a stable content-derived id. -- `AlignmentSample`: logical tensors, selected logprobs, and actual runtime provenance. -- `AlignmentResult`: global/per-rank drift, pass/fail, actual applied knobs, and optional - cost metrics. -- `ResultCube`: results indexed by normalized knob values, independent of execution order. - -Every `KnobDefinition` must declare: - -```text -name: - stable dotted name, for example rollout.tensor_parallel_size - -source_class: - logical-layout | arithmetic | reduction | representation | execution - -lifecycle: - request | engine-construction | process-start | build - -targets: - rollout | training | both | kernel - -domain: - allowed typed values - -capability: - how an adapter proves that a value is supported - -constraints: - incompatible or conditional combinations - -apply: - engine-specific materialization hook - -provenance: - how the actual applied value is read back and reported -``` - -The controller must compare requested and actual provenance. A silent runtime fallback is -an invalid ablation unless the fallback itself is the declared knob under test. - -### Grid Composition - -`GridPlanner` should support the following modes over the same typed axes: - -- `product`: full Cartesian grid; -- `one_at_a_time`: baseline plus one changed factor per case; -- `pairwise`: covering pairs without requiring the full Cartesian product; -- `zip`: paired values such as compatible model/dtype artifacts; -- fixed overrides and named slices; -- capability and compatibility constraints; -- deterministic case ids, filtering, resume, and retry. - -A normal workflow starts with `one_at_a_time`, expands to `pairwise` only when single -factors do not explain the failure, and uses `product` for an explicit grid search. CI -runs a named slice of the same definition rather than maintaining a separate handwritten -test matrix. - -For every generated case, `IsolationValidator` compares its normalized spec with the -baseline and rejects undeclared changes. This is what makes an arithmetic-only, -reduction-only, or quantization-only claim trustworthy. - -### Minimal Sufficient Alignment Analysis - -The analyzer treats "align this knob between rollout and training" as an intervention. It -reports the smallest passing intervention set found by the executed grid: - -```text -production mismatch: - fail - -align attention backend only: - fail - -align logp reduction only: - pass - -minimal sufficient alignment candidate: - {logp.reduction_policy} - -unrelated differences left enabled: - attention backend, cache policy, TP/FSDP topology -``` - -This result is evidence for the smallest effective intervention, not automatic proof of -root cause. A later fix PR still needs the smallest reproducer and a local regression. - -## Mapping to Current Code - -The controller should initially map to existing configuration surfaces instead of -introducing a second execution stack. - -| High-level knob | Current code path | Required adapter behavior | -| --- | --- | --- | -| `rollout.tensor_parallel_size` | `VLLMSamplerConfig.engine_kwargs` | Materialize `tensor_parallel_size` before vLLM engine construction and read it back from runtime metadata. | -| `rollout.dtype` | `VLLMSamplerConfig.engine_kwargs["dtype"]` | Normalize string/torch dtype and record the actual engine dtype. | -| `sampling.temperature` | `VLLMSamplerConfig.sampling_params` | Apply per request; require the same scoring semantics on both sides. | -| `execution.prefix_cache` | `VLLMSamplerConfig.enable_prefix_caching` | Treat as engine-construction-time, not a request toggle. | -| `training.attention_backend` | `StatelessForwardConfig.attention_backend` | Apply before forward and report requested backend plus any actual fallback. | -| `training.output_dtype` | `StatelessForwardConfig.output_dtype` | Keep observation dtype separate from model compute dtype. | -| `training.compute_dtype` | `TorchRLTrainingConfig.dtype` and FSDP model construction | Materialize before wrapping/sharding the model. | -| `logp.backend` | `RolloutExecutor` / `TorchRLTrainingConfig.logp_backend` | Reuse `resolve_logp_op_type()` aliases and report the resolved op type and concrete backend class. | -| `logp.deterministic` | `require_batch_invariant_logp` | Express policy intent; do not hard-code a CUDA implementation in the controller. | -| `training.sharding` | new score-only FSDP adapter | Materialize world size and sharding strategy before process-group/model construction. | -| `logp.tp_layout` | `linear_logp` `tp_group`, `vocab_start_index`, `global_vocab_size` | Record shard boundaries and reject incomplete ownership metadata. | -| `kernel.fast_math` | `KERNEL_ALIGN_USE_FAST_MATH` | Treat as build-time and bind the case to a distinct built artifact. | -| `kernel.sm90_path` | `KERNEL_ALIGN_FORCE_SM90` and compiled extension | Capability-gate by architecture and build artifact; never switch it after import. | - -The existing `KernelRegistry` caches instances and resolves priority maps during -initialization. vLLM TP, dtype, and prefix caching also belong to engine construction. -Therefore the runner must not mutate these values in a long-lived process and assume the -next case is isolated. - -Cases may share a worker only when their engine-construction and process-start -fingerprints are identical. Request-time knobs may reuse that worker. Build-time knobs -always select a prebuilt artifact and a separate process. The artifact id and extension -build metadata are part of result provenance. - -## Kernel Integration Contract - -Kernel work may require a new or rewritten implementation, but the ablation controller -must not know CUDA/Triton class names or kernel launch details. - -The kernel boundary should expose a backend descriptor with: - -- stable backend id and semantic operator name; -- supported device architectures, dtypes, shapes, and parallel layouts; -- determinism/alignment properties; -- required TP/SP metadata and collectives; -- configuration lifecycle, including build-time flags; -- concrete implementation selected at runtime; -- fallback behavior; -- version/build fingerprint. - -The controller requests a policy such as `production`, `reference`, `deterministic`, or a -stable backend id. The kernel adapter resolves that policy through `KernelRegistry` and -records the concrete implementation. Strict WS2 cases reject an undeclared fallback. - -A rewritten kernel integrates cleanly by: - -1. implementing the existing operator semantic interface; -2. registering a new stable backend descriptor; -3. passing #108 operator accuracy and batch-invariance checks; -4. declaring TP/SP metadata and supported lifecycle knobs; -5. adding one isolated end-to-end controller case; -6. reporting performance/memory overhead against the production backend. - -It should not require a new branch in `GridPlanner`. If a framework cannot inject the -kernel through a supported hook, its engine adapter reports the knob as unsupported; it -must not claim that the ablation ran. - -## Repository Fit - -The current repository already provides useful pieces: - -- #108 owns `tolerance_contract.json`. -- `VLLMSamplerConfig` exposes loose `engine_kwargs`, `sampling_params`, and prefix-cache - configuration. -- `StatelessForwardConfig` exposes attention backend, temperature, and output dtype. -- `TorchRLTrainingConfig` exposes compute dtype, `logp_backend`, and the deterministic - requirement. -- `resolve_logp_op_type()` already separates user-facing logp policy from registry op type. -- TP `linear_logp` already accepts explicit process group and vocab-shard metadata. -- `RolloutStageResult` and the weight bridge carry iteration/weight version. -- `StatelessForwardExecutor` is a reusable no-update teacher-forcing scorer. - -The missing pieces are the typed experiment model, actual-value provenance, strict scoring -payload, FSDP score-only adapter, lifecycle-aware knob materializers, grid planner, and -result cube. - -`DeepSpeedTrainingWorker.train()` still performs backward/step and constructs its current -objective's `old_logps` from recomputed values. It is not a WS2 comparator. A later -DeepSpeed scorer must be a separate read-only adapter. - -## Ownership Boundaries - -| Issue | Boundary | -| --- | --- | -| [#108](https://github.com/RL-Align/RL-Kernel/issues/108) | Owns numerical thresholds. | -| [#109](https://github.com/RL-Align/RL-Kernel/issues/109) | Owns deterministic TP reduction implementations. | -| [#110](https://github.com/RL-Align/RL-Kernel/issues/110) | Owns SP-aware operators and reductions. | -| [#112](https://github.com/RL-Align/RL-Kernel/issues/112) | Owns deterministic collective implementations. | -| [#113](https://github.com/RL-Align/RL-Kernel/issues/113) | Owns the later distributed forward/backward chain gate. | -| [#116](https://github.com/RL-Align/RL-Kernel/issues/116) | Shares the tolerance/report foundation implemented by B1. | -| [#127](https://github.com/RL-Align/RL-Kernel/issues/127) | Owns the pinned multi-GPU dual-engine environment. | -| [#130](https://github.com/RL-Align/RL-Kernel/issues/130) | Owns full FSDP/Megatron training integration and backward. | -| [#131](https://github.com/RL-Align/RL-Kernel/issues/131) | Owns the later production cross-benchmark command. | -| [#136](https://github.com/RL-Align/RL-Kernel/issues/136) | Owns automatic layer-wise probes. | - -## Revised Modular PR Roadmap - -The identifiers below are roadmap labels, not existing GitHub PR numbers. The former -Phases A, B, and C are consolidated because they jointly form the baseline infrastructure. - -### Phase 1: Baseline Infrastructure - -#### B1 — Alignment contract, comparator, and report - -**Scope:** Expose #108 tolerance lookup; add canonical identity/provenance/sample types, -logical comparability validation, active-token drift metrics, and one JSON/human report. - -**Acceptance:** CPU tests cover identity mismatch, masks, percentiles, zero active tokens, -worst-token metadata, and dtype-specific pass/fail without copying threshold values. - -**Why one PR:** These types form one public contract and cannot provide useful independent -behavior when landed separately. - -#### B2 — Exact rollout and teacher-forcing scoring adapters - -**Scope:** Normalize vLLM sampled-token logprobs and rollout provenance; add strict -rollout-to-teacher-forcing collation; define the read-only scorer protocol and adapt -`StatelessForwardExecutor`. - -**Acceptance:** A fixture round trip preserves prompt/generated ids, masks, selected -logprobs, weight version, and available position metadata. Missing identity data or -undeclared backend fallback fails explicitly. Repeated scoring does not change model state. - -**Non-goal:** No FSDP, subprocess runner, or grid planner. - -#### B3 — Score-only FSDP adapter and baseline controls - -**Scope:** Add a PyTorch FSDP scorer with no optimizer/backward, then add A0 identical -stateless scoring and unsharded-vs-FSDP controls. - -**Acceptance:** A0 passes on CPU; a labeled two-GPU/NCCL control proves FSDP recomputation -is clean and model state is unchanged. - -**Non-goal:** Full training integration remains in #130. - -#### B4 — Paired runner, artifacts, and rank aggregation - -**Scope:** Launch rollout/training scorers with independent world sizes; write versioned -canonical artifacts; enforce timeout/cleanup; aggregate deterministic per-rank/global -reports. - -**Acceptance:** CPU fixtures cover child failure, timeout, malformed artifact, duplicate -or missing ranks, weight-version mismatch, and global worst-token selection. - -**Design requirement:** The runner accepts separate construction/process/build -fingerprints so the later controller can isolate cases correctly. - -### Phase 2: Composable Ablation Controller - -#### C1 — Typed experiment model, knob registry, and grid planner - -**Scope:** Implement `ExperimentDefinition`, typed knob descriptors, constraints, -capability declarations, stable case ids, and `product`, `one_at_a_time`, `pairwise`, and -`zip` planners. - -**Acceptance:** Pure CPU tests generate deterministic grids, reject invalid combinations, -resume by case id, and prove each one-at-a-time case changes exactly one declared knob. - -**Non-goal:** Do not launch engines in this PR. - -#### C2 — Runtime knob materializers and capability checks - -**Scope:** Map controller knobs to current vLLM, stateless, and FSDP configuration -surfaces. Separate request-time, engine-construction, and process-start application. Read -back actual values and construction fingerprints. - -**Acceptance:** Fake engine adapters prove every requested value is either applied and -reported or rejected as unsupported. No silent fallback is accepted in strict cases. - -#### C3 — Kernel policy bridge - -**Scope:** Add the backend descriptor and kernel materializer boundary described above. -Adapt existing logp policy aliases and TP metadata without changing kernel math. - -**Acceptance:** The same experiment definition can select production/reference/ -deterministic logp policies and report the concrete registry backend. A fake rewritten -kernel registers without a controller code change. - -**Non-goal:** Kernel rewrites discovered later remain one-root-cause fix PRs. - -#### C4 — Grid executor and result cube - -**Scope:** Execute C1 cases through B4, pool only workers with identical lifecycle -fingerprints, select build artifacts, persist results, and expose filtering/resume plus a -machine-readable result cube. - -**Acceptance:** An interrupted fake grid resumes without rerunning completed cases; -requested and actual provenance are queryable for every axis. - -### Phase 3: Core Scenario and Minimal Alignment - -#### M1 — TP=2 rollout versus FSDP diagnostic grid - -**Scope:** Define the first real experiment using the controller: fixed model/tokenizer/ -tokens, vLLM TP=2 rollout, FSDP recomputation, bf16, and production defaults. Generate the -production point plus one-at-a-time alignment interventions. - -**Acceptance:** Execution and reports succeed on the pinned #127 environment. Numerical -failure is recorded without weakening #108. - -#### M-FIX-N — One minimal root cause per PR - -Each fix PR consumes the smallest controller case that exposes one problem. A kernel -rewrite, collective change, metadata fix, or adapter fix remains separate. - -A fix must show: - -- the failing production or isolated case; -- the smallest intervention that makes it pass; -- one local implementation change; -- A0 and unrelated-knob regressions; -- actual backend provenance; -- performance/memory cost when applicable. - -#### M2 — Promote the minimally aligned core case to a gate - -**Scope:** After required M-FIX PRs, gate TP=2/FSDP using the smallest passing alignment -set, not a fully reference configuration. - -**Acceptance:** The report names which knobs were aligned, which differences remained -enabled, and why a broader alignment level was unnecessary. - -### Phase 4: Grid Coverage and Ablation Closure - -#### G1 — Required composable grid - -**Scope:** Add the required batch-size, padding/layout, dtype, and cache/position axes as -declarative knob values and constraints. Allow full product, named slices, and -one-at-a-time views from the same definition. - -**Acceptance:** A user can request, for example: - -```text -batch_size = [1, 8] -padding_side = [left, right] -dtype = [fp32, bf16, fp16] -prefix_cache = [off, on] -logp.backend = [production, deterministic] -``` - -without writing a new test function. Unsupported combinations are capability-filtered -with explicit reasons, and every result is indexed in the same cube. - -#### G2 — A0-A5 profile and minimal-alignment wrapper - -**Scope:** Express the RFC's A0-A5 ablations as presets over C1 rather than separate test -implementations: - -- A0: fully aligned diagnostic reference; -- A1: arithmetic one-at-a-time; -- A2: reduction/topology one-at-a-time; -- A3: representation/quantization one-at-a-time; -- A4: pairwise expansion only when needed; -- A5: production mismatch. - -Add a CLI/config wrapper that selects profiles, axes, filters, and output location. - -**Acceptance:** Phase F behavior is only a planner/profile layer over G1. It adds no -engine-specific branching. - -#### G3 — Targeted GPU CI and downstream handoff - -**Scope:** Run a curated named slice of the same grid in labeled GPU CI, upload the result -cube, and expose fixtures/reports to #113/#131. - -**Acceptance:** CI distinguishes launch/environment/numerical failure, always cleans up, -and does not maintain a second handwritten matrix. - -## PR Sizing Rules - -The consolidated roadmap uses fewer baseline PRs, but later numerical fixes remain small: - -1. B1-B4 may each land one cohesive baseline subsystem. -2. C1-C4 each own one controller layer: planning, runtime materialization, kernel policy, - or execution/results. -3. Adding a new ordinary knob changes one descriptor and one engine adapter, not the - planner. -4. Adding a rewritten kernel changes the kernel implementation and its backend descriptor, - not the controller. -5. M-FIX PRs contain one root cause only. -6. G1/G2 add declarative grids/profiles and must not include numerical fixes. -7. No PR adds or copies a tolerance value. - -## Completion Criteria for #111 - -#111 is complete when: - -1. semantic identity validation is strict and independent of numerical alignment; -2. the controller can compose, filter, resume, and report a multidimensional configuration - grid; -3. requested knobs are verified against actual runtime/kernel provenance; -4. the TP=2/FSDP production case is brought into #108 contract using the documented - smallest sufficient alignment set; -5. batch, padding/layout, dtype, and cache/position axes are available through G1 without - new test functions; -6. at least one production, one arithmetic, and one reduction/topology slice run through - the same controller/report path; -7. a rewritten kernel can register through C3 without changing grid-planner code; -8. the stable core and selected grid slice run in targeted GPU CI; -9. forward fixtures and result cubes are reusable by #113 and #131. - -SP, additional TP sizes, quantization variants, exhaustive product grids, and downstream -training effects remain extensions unless maintainers promote named grid slices into the -required gate. Full/bitwise internal alignment is not a completion criterion unless a -separate contract explicitly requires it.